00000100:00100000:0.0:1761315077.851966:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315077.851978:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322470464:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315077.851993:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315077.852016:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470208 00000100:00100000:0.0:1761315077.853080:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470528 00000100:00100000:0.0:1761315077.853109:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470592 00000100:00100000:0.0:1761315077.853122:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3957 00000100:00100000:0.0:1761315077.853127:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322470208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853163:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322470208:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (1237us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315077.853169:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3957 00000100:00100000:0.0:1761315077.853182:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3958 00000100:00100000:0.0:1761315077.853185:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a597521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322470528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853206:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a597521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322470528:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (1217us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315077.853212:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3958 00000100:00100000:0.0:1761315077.853222:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3959 00000100:00100000:0.0:1761315077.853224:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5e3a129 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322470592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853245:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5e3a129 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322470592:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (1215us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315077.853249:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3959 00000100:00100000:0.0:1761315077.853269:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470336 00000100:00100000:0.0:1761315077.853289:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2487 00000100:00100000:0.0:1761315077.853292:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010d50b50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322470336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853315:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010d50b50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322470336:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (337us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315077.853321:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2487 00000100:00100000:0.0:1761315077.853377:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322470592:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853387:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322470528:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853395:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a82025fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322470208:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315077.853440:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470400 00000100:00100000:0.0:1761315077.853458:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470464 00000100:00100000:0.0:1761315077.853469:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1940 00000100:00100000:0.0:1761315077.853473:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322470400:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315077.853559:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315077.853601:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322470400:12345-0@lo:8:kworker/u8:0.0 Request processed in 128us (1635us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315077.853606:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1940 00000100:00100000:0.0:1761315077.853616:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1941 00000100:00100000:0.0:1761315077.853619:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322470464:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315077.853695:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315077.853714:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322470464:12345-0@lo:8:kworker/u8:0.0 Request processed in 94us (1721us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315077.853720:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1941 00000100:00080000:0.0:1761315077.853748:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315077.853751:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315077.853754:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322470400:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315077.853761:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315077.853763:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315077.853765:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322470464:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315078.876510:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315079.880423:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315079.882616:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315079.882957:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315079.884402:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315079.884432:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307530816 00000100:00100000:0.0:1761315079.884460:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1942 00000100:00100000:0.0:1761315079.884465:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307530816:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315079.886335:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315079.886377:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307530816:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 1911us (1979us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315079.886384:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1942 00010000:00010000:0.0:1761315079.899752:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315081.947496:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315082.971499:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315082.974606:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322470784:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315082.977674:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315082.977679:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315082.977682:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315082.977688:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3325 00000100:00080000:0.0:1761315082.977691:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315082.977696:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315082.977717:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000036336669 x1846869322471168/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315082.977733:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315082.977737:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315082.977739:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315082.977742:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3325 00000100:00080000:0.0:1761315082.977744:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315082.977750:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315082.977762:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000572cc7ae x1846869322471232/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315082.977807:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322471168:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315082.977844:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315082.977864:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322471232:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315082.977906:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315082.977942:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471168 00000100:00100000:0.0:1761315082.977974:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471232 00000100:00100000:0.0:1761315082.977990:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1943 00000100:00100000:0.0:1761315082.977999:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322471168:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315082.978101:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315082.978144:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322471168:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315082.978157:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1943 00000100:00100000:0.0:1761315082.978175:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471424 00000100:00100000:0.0:1761315082.978203:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1338 00000100:00100000:0.0:1761315082.978211:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322471424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.978246:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322471424:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (303us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315082.978253:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1338 00000100:00100000:0.0:1761315082.978265:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:0.0:1761315082.978269:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322471232:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315082.978371:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315082.978398:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322471232:12345-0@lo:8:kworker/u8:0.0 Request processed in 128us (493us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315082.978405:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00000100:00100000:0.0:1761315082.978432:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471040 00000100:00100000:0.0:1761315082.978461:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322470912 00000100:00100000:0.0:1761315082.979836:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471360 00000100:00100000:0.0:1761315082.979862:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3963 00000100:00100000:0.0:1761315082.979870:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a919a5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322471040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980066:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a919a5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322471040:12345-0@lo:400:kworker/u8:0.0 Request processed in 198us (2172us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315082.980077:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3963 00000100:00100000:0.0:1761315082.980097:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3966 00000100:00100000:0.0:1761315082.980101:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322471296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980128:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322471296:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (2100us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315082.980140:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3966 00000100:00080000:0.0:1761315082.980195:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315082.980199:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315082.980203:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322471168:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315082.980213:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315082.980216:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315082.980218:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322471232:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980303:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322471040:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980314:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322471424:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980323:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322471104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980333:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322470848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980393:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322470976:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980403:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b6ed247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322470912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980411:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004131e225 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322471296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980674:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a82025fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322471360:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315082.980768:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322471488:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315082.980795:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315082.987239:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322471488:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315085.002950:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531008 00000100:00100000:0.0:1761315085.002981:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 3967 00000100:00100000:0.0:1761315085.002986:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000047431c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307530944:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315085.003027:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000047431c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307530944:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 41us (1258us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315085.003035:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 3967 00000100:00100000:0.0:1761315085.003046:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 3968 00000100:00100000:0.0:1761315085.003048:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307531008:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315085.003069:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307531008:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 19us (1081us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315085.003075:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 3968 00010000:00010000:0.0:1761315085.020305:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315086.043483:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315088.093263:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315088.093679:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322471616 00000100:00100000:0.0:1761315088.093704:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 896 00000100:00100000:0.0:1761315088.093721:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf3ad421 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322471616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315088.093740:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315088.093767:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf3ad421 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322471616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (442us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315088.093772:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 896 00000100:00100000:0.0:1761315088.093964:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322472000:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315088.093985:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315088.093997:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322472064:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315088.094016:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315088.094047:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472000 00000100:00100000:0.0:1761315088.094063:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472064 00000100:00100000:0.0:1761315088.094075:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1761315088.094077:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003a7dc6c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322472000:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315088.094252:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315088.095890:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003a7dc6c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322472000:12345-0@lo:8:kworker/u8:2.0 Request processed in 1808us (1901us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315088.095899:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1761315088.098950:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315088.098971:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008824de23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322472320:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315088.099038:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315089.118767:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315090.122593:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531136 00000100:00100000:0.0:1761315090.122655:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1493 00000100:00100000:0.0:1761315090.122663:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307531136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315090.122743:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307531136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 80us (1484us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315090.122751:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1493 00000100:00100000:0.0:1761315090.126085:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531328 00000100:00100000:0.0:1761315090.126126:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1948 00000100:00100000:0.0:1761315090.126130:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307531328:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315090.126221:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315090.126266:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307531328:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 134us (2306us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315090.126273:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1948 00000100:00100000:0.0:1761315090.652517:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322472384:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315091.163578:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315092.187699:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315093.213170:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322472640:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.213198:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315093.213214:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322473024:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.213232:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315093.213244:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315093.213248:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322472576:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.213265:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315093.213276:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322472960:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.213289:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315093.215301:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472896 00000100:00100000:0.0:1761315093.215332:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472640 00000100:00100000:0.0:1761315093.215361:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472576 00000100:00100000:0.0:1761315093.215373:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322472960 00000100:00100000:0.0:1761315093.215384:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3975 00000100:00100000:0.0:1761315093.215388:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322472896:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215432:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322472896:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (2742us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315093.215438:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3975 00000100:00100000:0.0:1761315093.215451:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3976 00000100:00100000:0.0:1761315093.215454:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322472640:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215494:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322472640:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (2296us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315093.215500:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3976 00000100:00100000:0.0:1761315093.215512:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3977 00000100:00100000:0.0:1761315093.215515:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322472576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215538:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322472576:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (2272us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315093.215543:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3977 00000100:00100000:0.0:1761315093.215552:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3978 00000100:00100000:0.0:1761315093.215555:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093b81021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322472960:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215577:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093b81021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322472960:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (2287us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315093.215583:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3978 00000100:00100000:0.0:1761315093.215644:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322472576:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215654:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322472960:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215663:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322472640:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215723:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ac61232 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322472896:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215733:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e0281a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322473088:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315093.215742:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322472512:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315095.241964:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531392 00000100:00100000:0.0:1761315095.242006:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1495 00000100:00100000:0.0:1761315095.242011:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307531392:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315095.242054:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307531392:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 42us (798us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315095.242061:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1495 00010000:00010000:0.0:1761315095.259585:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315098.331780:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315098.331824:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315098.331828:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315098.331831:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315098.331836:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3340 00000100:00080000:0.0:1761315098.331839:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315098.331843:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315098.331862:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000085790e43 x1846869322473664/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315098.331878:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315098.331882:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315098.331883:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315098.331887:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3340 00000100:00080000:0.0:1761315098.331889:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315098.331891:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315098.331901:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007c685a75 x1846869322473728/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315100.361581:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531840 00000100:00100000:0.0:1761315100.361615:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1954 00000100:00100000:0.0:1761315100.361620:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307531840:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315100.361708:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315100.361753:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307531840:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 132us (1035us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315100.361759:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1954 00000100:00100000:0.0:1761315100.893040:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322474048:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315103.451992:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322474304:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315103.452032:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315103.452056:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315103.452065:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004070f96b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322474240:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315103.452096:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315103.452118:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322474496:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315103.452161:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315103.452192:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322474688:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315103.452223:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315103.453231:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc64cfd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322474112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (1444us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315103.453239:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 899 00000100:00100000:3.0:1761315103.453259:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322474560:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315103.453282:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315103.453295:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322474624:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315103.453314:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315103.453375:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322474560 00000100:00100000:3.0:1761315103.453392:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322474624 00000100:00100000:3.0:1761315103.453416:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1955 00000100:00100000:3.0:1761315103.453419:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322474560:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:3.0:1761315103.453510:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315103.453545:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322474560:12345-0@lo:8:kworker/u8:1.0 Request processed in 123us (262us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315103.453553:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1955 00000100:00080000:3.0:1761315103.453588:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315103.453592:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315103.453599:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322474560:0@lo:8:kworker/u8:1.0 00000100:00080000:3.0:1761315103.453782:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315103.453785:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315103.453788:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322474624:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315103.454605:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322474688:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315103.454623:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004070f96b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322474240:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315103.454634:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322474496:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315103.454645:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322474304:0@lo:400:kworker/u8:1.0 00010000:00010000:3.0:1761315104.476431:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315105.480840:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307531904 00000100:00100000:3.0:1761315105.480873:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1499 00000100:00100000:3.0:1761315105.480878:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307531904:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315105.480920:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307531904:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 40us (518us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315105.480926:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1499 00000100:00100000:3.0:1761315106.015639:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322474880:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315106.015705:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761315106.526060:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315107.548243:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315108.573536:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315108.573542:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315108.573545:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315108.573552:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3351 00000100:00080000:3.0:1761315108.573555:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315108.573560:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315108.573578:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275a380d x1846869322475392/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315108.573592:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315108.573595:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315108.573596:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315108.573599:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3351 00000100:00080000:3.0:1761315108.573601:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315108.573604:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315108.573614:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001c97fc10 x1846869322475456/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315108.573659:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322474944 00000100:00100000:3.0:1761315108.573698:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 900 00000100:00100000:3.0:1761315108.573704:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000222cd7fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322474944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315108.573728:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315108.573781:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000222cd7fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322474944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (2017us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315108.573787:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 900 00000100:00100000:3.0:1761315108.573809:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322475072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.573841:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.573852:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322475328:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.573872:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.573891:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315108.573896:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322475008:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315108.573917:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.573928:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322475264:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.573947:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.573961:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322475648:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.573979:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.573999:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322475392:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574017:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.574028:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322475456:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574046:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315108.574074:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475328 00000100:00100000:3.0:1761315108.574096:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475008 00000100:00100000:3.0:1761315108.574110:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2507 00000100:00100000:3.0:1761315108.574113:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322475328:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574146:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322475328:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315108.574151:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2507 00000100:00100000:3.0:1761315108.574162:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2508 00000100:00100000:3.0:1761315108.574165:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322475008:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315108.574184:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315108.574210:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322475008:12345-0@lo:41:osp-pre-0-1.0 Request processed in 44us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315108.574215:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2508 00000100:00100000:3.0:1761315108.574232:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475264 00000100:00100000:3.0:1761315108.574253:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3993 00000100:00100000:3.0:1761315108.574256:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322475264:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574281:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322475264:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315108.574286:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3993 00000100:00100000:3.0:1761315108.574305:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475648 00000100:00100000:3.0:1761315108.574324:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475392 00000100:00100000:3.0:1761315108.574334:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475456 00000100:00100000:3.0:1761315108.574394:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1345 00000100:00100000:3.0:1761315108.574398:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322475648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574429:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322475648:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (449us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315108.574433:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1345 00000100:00100000:3.0:1761315108.574444:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1761315108.574447:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322475392:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315108.574529:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315108.574557:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322475392:12345-0@lo:8:kworker/u8:0.0 Request processed in 108us (539us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315108.574562:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1761315108.574572:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1959 00000100:00100000:3.0:1761315108.574575:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322475456:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315108.574647:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315108.574668:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322475456:12345-0@lo:8:kworker/u8:0.0 Request processed in 92us (622us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315108.574673:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1959 00000100:00080000:3.0:1761315108.574703:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315108.574707:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315108.574710:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322475392:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315108.574717:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315108.574719:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315108.574721:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322475456:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574767:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322475008:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315108.574775:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322475264:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574782:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322475648:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315108.574789:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322475328:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315109.596321:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315110.601662:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315110.601732:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315110.601745:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532352 00000100:00100000:0.0:1761315110.601788:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1960 00000100:00100000:0.0:1761315110.601795:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307532352:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315110.601890:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315110.601951:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307532352:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 156us (297us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315110.601961:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1960 00000100:00100000:3.0:1761315110.606934:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315110.612926:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00010000:00010000:3.0:1761315110.620005:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315111.131890:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475712 00000100:00100000:3.0:1761315111.133520:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2510 00000100:00100000:3.0:1761315111.133530:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322475712:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315111.133569:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315111.133648:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322475712:12345-0@lo:41:osp-pre-1-0.0 Request processed in 118us (1941us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315111.133655:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2510 00000100:00100000:0.0:1761315113.691542:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322475776:0@lo:400:kworker/u8:1.0 00000100:00080000:3.0:1761315113.691556:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315113.691561:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315113.691564:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315113.691569:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3356 00000100:00080000:3.0:1761315113.691572:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315113.691577:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315113.691597:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000abf11666 x1846869322476096/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315113.691612:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315113.691617:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315113.691618:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315113.691620:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315113.691624:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3356 00000100:00080000:3.0:1761315113.691626:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315113.691628:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315113.691636:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008bbe5509 x1846869322476160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315113.691657:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322476032:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.691677:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322475968:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.691684:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315113.691702:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322476416:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.691715:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315113.691731:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315113.691738:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322476352:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.691750:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315113.691758:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011e09f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322475840:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.691767:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315113.691788:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315113.691791:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315113.691799:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322475904:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.691806:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322476224:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.691827:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315113.691835:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315113.691847:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322476288:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.691869:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475776 00000100:00100000:3.0:1761315113.691874:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315113.691907:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475968 00000100:00100000:0.0:1761315113.691914:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1502 00000100:00100000:0.0:1761315113.691920:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322475776:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.691937:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475840 00000100:00100000:3.0:1761315113.691960:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476224 00000100:00100000:0.0:1761315113.691965:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322475776:12345-0@lo:400:kworker/u8:1.0 Request processed in 44us (352us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.691972:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1502 00000100:00100000:3.0:1761315113.691972:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476288 00000100:00100000:3.0:1761315113.691991:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3999 00000100:00100000:3.0:1761315113.691996:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322475968:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.692006:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476416 00000100:00100000:0.0:1761315113.692026:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476352 00000100:00100000:0.0:1761315113.692047:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1346 00000100:00100000:3.0:1761315113.692047:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322475968:12345-0@lo:400:kworker/u8:1.0 Request processed in 47us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.692055:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322476416:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.692059:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3999 00000100:00100000:3.0:1761315113.692076:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4000 00000100:00100000:3.0:1761315113.692080:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322475840:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.692094:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322476416:12345-0@lo:400:kworker/u8:1.0 Request processed in 40us (364us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.692100:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1346 00000100:00100000:0.0:1761315113.692121:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1347 00000100:00100000:3.0:1761315113.692121:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322475840:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (330us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.692128:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322476352:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.692133:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4000 00000100:00100000:3.0:1761315113.692150:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4001 00000100:00100000:3.0:1761315113.692156:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4e2e24c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322476224:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.692160:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322476352:12345-0@lo:400:kworker/u8:1.0 Request processed in 30us (392us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.692166:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1347 00000100:00100000:3.0:1761315113.692191:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4e2e24c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322476224:12345-0@lo:400:kworker/u8:1.0 Request processed in 35us (357us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315113.692198:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4001 00000100:00100000:0.0:1761315113.692199:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476032 00000100:00100000:0.0:1761315113.692219:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322475904 00000100:00100000:3.0:1761315113.692226:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4002 00000100:00100000:3.0:1761315113.692232:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322476288:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315113.692237:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2511 00000100:00100000:0.0:1761315113.692242:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322476032:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.692262:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322476288:12345-0@lo:400:kworker/u8:1.0 Request processed in 28us (388us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315113.692270:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4002 00000100:00100000:0.0:1761315113.692281:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322476032:12345-0@lo:400:kworker/u8:1.0 Request processed in 37us (597us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315113.693229:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2511 00010000:00010000:0.0:1761315113.693275:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315113.693301:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476480 00000100:00100000:3.0:1761315113.695746:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2512 00000100:00100000:3.0:1761315113.695755:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322475904:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315113.695833:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322475904:12345-0@lo:400:kworker/u8:1.0 Request processed in 79us (4006us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315113.695840:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2512 00000100:00100000:3.0:1761315115.722899:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315115.722978:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315115.723082:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315115.725514:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4004 00000100:00100000:0.0:1761315115.725521:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001a76876 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307532480:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315115.726520:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001a76876 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307532480:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 995us (3435us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315115.726533:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4004 00000100:00100000:0.0:1761315115.727278:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532608 00000100:00100000:0.0:1761315115.727330:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1963 00000100:00100000:0.0:1761315115.727337:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307532608:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315115.727451:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315115.727518:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307532608:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 180us (320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315115.727525:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1963 00010000:00010000:3.0:1761315115.739547:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315116.252675:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476608 00000100:00100000:0.0:1761315116.252728:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2514 00000100:00100000:0.0:1761315116.252734:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322476608:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315116.252768:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315116.252819:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322476608:12345-0@lo:41:osp-pre-1-0.0 Request processed in 84us (846us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315116.252826:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2514 00010000:00010000:0.0:1761315116.763479:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315118.811514:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315118.811518:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322476672:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315118.811577:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315118.811784:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322476736:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.811827:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315118.811846:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322476992:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.811877:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315118.812968:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322477376:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.813011:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315118.814115:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476736 00000100:00100000:3.0:1761315118.814147:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1504 00000100:00100000:3.0:1761315118.814151:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322476736:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.814184:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322476736:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (2362us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315118.814190:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1504 00000100:00100000:3.0:1761315118.814226:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476800 00000100:00100000:3.0:1761315118.814250:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 902 00000100:00100000:3.0:1761315118.814254:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322476800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315118.814274:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315118.814322:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322476800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (2359us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315118.814330:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 902 00000100:00100000:3.0:1761315118.814385:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322477376:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.814394:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322476736:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.814434:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322476992 00000100:00100000:3.0:1761315118.814468:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2515 00000100:00100000:3.0:1761315118.814474:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ebd9dcb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322476672:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315118.814498:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315118.814543:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ebd9dcb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322476672:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (2964us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315118.814550:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2515 00000100:00100000:3.0:1761315118.814567:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2516 00000100:00100000:3.0:1761315118.814572:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322476992:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.814612:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322476992:12345-0@lo:400:kworker/u8:1.0 Request processed in 39us (2738us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315118.814619:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2516 00000100:00100000:3.0:1761315118.814652:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322476928:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315118.814669:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322476672:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315118.814708:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322476992:0@lo:400:kworker/u8:1.0 00010000:00010000:0.0:1761315119.836157:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315120.841027:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315120.841078:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315120.841114:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315120.841140:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315120.841184:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532736 00000100:00100000:3.0:1761315120.841221:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532800 00000100:00100000:3.0:1761315120.841243:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4009 00000100:00100000:3.0:1761315120.841249:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307532736:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315120.841387:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532672 00000100:00100000:0.0:1761315120.841425:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1505 00000100:00100000:0.0:1761315120.841430:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307532672:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315120.841792:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307532736:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 542us (677us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315120.841800:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4009 00000100:00100000:0.0:1761315120.842083:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307532672:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 652us (1063us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315120.842091:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1505 00010000:00010000:3.0:1761315121.886499:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315122.908334:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315123.931593:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322477568:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315123.931651:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315123.931687:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322477824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315123.931710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315123.931733:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322478208:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315123.931752:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315123.931765:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315123.931769:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322477632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315123.931789:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315123.931803:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322478016:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315123.931819:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315123.931860:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322477888 00000100:00100000:3.0:1761315123.931878:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1967 00000100:00100000:3.0:1761315123.931881:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322477888:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315123.931970:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315123.932014:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322477888:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (277us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315123.932019:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1967 00000100:00100000:3.0:1761315123.932030:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478208 00000100:00100000:3.0:1761315123.932047:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322477952 00000100:00100000:3.0:1761315123.932060:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1968 00000100:00100000:3.0:1761315123.932063:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322477952:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315123.932153:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315123.932172:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322477952:12345-0@lo:8:kworker/u8:0.0 Request processed in 108us (403us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315123.932178:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1968 00000100:00100000:3.0:1761315123.932209:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322477696 00000100:00100000:0.0:1761315123.932223:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478080 00000100:00100000:3.0:1761315123.932232:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4013 00000100:00100000:3.0:1761315123.932236:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322477696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315123.932253:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4014 00000100:00100000:0.0:1761315123.932270:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322478080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315123.932277:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322477696:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (148us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315123.932284:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4013 00000100:00100000:0.0:1761315123.932318:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322478080:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (158us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315123.932327:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4014 00000100:00100000:0.0:1761315123.932387:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322477504 00000100:00100000:0.0:1761315123.932419:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1506 00000100:00100000:0.0:1761315123.932424:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322477504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315123.932468:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322477504:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315123.932476:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1506 00000100:00100000:0.0:1761315123.932519:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2519 00000100:00100000:0.0:1761315123.932526:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322477760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315123.932566:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322477760:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315123.932576:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2519 00000100:00100000:0.0:1761315123.932619:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478144 00000100:00100000:0.0:1761315123.933475:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1351 00000100:00100000:0.0:1761315123.933480:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322478144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315123.933531:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322478144:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (1219us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315123.933538:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1351 00000100:00100000:3.0:1761315123.935692:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478272 00000100:00100000:3.0:1761315123.935728:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2520 00000100:00100000:3.0:1761315123.935733:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cdd78470 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322478272:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315123.935764:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315123.935814:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cdd78470 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322478272:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (292us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315123.935821:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2520 00000100:00100000:3.0:1761315125.967010:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315125.969260:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533120 00000100:00100000:3.0:1761315125.969423:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315125.969883:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315125.969913:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315125.970964:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307532928 00000100:00100000:0.0:1761315125.970989:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1507 00000100:00100000:0.0:1761315125.970993:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307532928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315125.971068:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4016 00000100:00100000:3.0:1761315125.971073:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf4ea074 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307533056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315125.971271:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307532928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 276us (1389us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315125.971279:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1507 00000100:00100000:3.0:1761315125.971708:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf4ea074 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307533056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 633us (1794us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315125.971715:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4016 00010000:00010000:0.0:1761315125.980040:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315126.491753:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478336 00000100:00100000:3.0:1761315126.491813:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2521 00000100:00100000:3.0:1761315126.491820:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322478336:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315126.491853:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315126.491909:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322478336:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (225us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315126.491917:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2521 00000100:00100000:0.0:1761315126.491952:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322478336:0@lo:41:osp-pre-1-0.0 00000100:00080000:0.0:1761315129.053153:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315129.053159:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315129.053162:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315129.053168:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3371 00000100:00080000:0.0:1761315129.053171:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315129.053177:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315129.053198:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008517e317 x1846869322478784/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315129.053214:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315129.053233:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315129.053234:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315129.053239:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3371 00000100:00080000:0.0:1761315129.053242:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315129.053247:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315129.053262:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005ee7850a x1846869322478848/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315129.053377:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478656 00000100:00100000:0.0:1761315129.053414:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322478976 00000100:00100000:0.0:1761315129.053441:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4017 00000100:00100000:0.0:1761315129.053449:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322478656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315129.053504:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322478656:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (227us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315129.053515:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4017 00000100:00100000:0.0:1761315129.053534:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4018 00000100:00100000:0.0:1761315129.053539:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322478976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315129.053570:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322478976:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (240us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315129.053576:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4018 00010000:00010000:0.0:1761315129.053594:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315129.053607:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1352 00000100:00100000:0.0:1761315129.053615:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322479040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315129.053644:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322479040:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (232us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315129.053650:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1352 00000100:00100000:0.0:1761315129.053664:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1971 00000100:00100000:0.0:1761315129.053667:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8081244 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322478848:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315129.054527:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322478592:0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761315129.054607:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315129.056286:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8081244 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322478848:12345-0@lo:8:kworker/u8:0.0 Request processed in 2611us (2767us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315129.056301:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1971 00000100:00100000:3.0:1761315131.084088:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315131.084132:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315131.084162:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533376 00000100:00100000:0.0:1761315131.084199:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1972 00000100:00100000:0.0:1761315131.084234:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307533376:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315131.084331:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315131.084421:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307533376:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 186us (337us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315131.084429:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1972 00000100:00100000:3.0:1761315131.084443:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315131.084475:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533248 00000100:00100000:3.0:1761315131.084507:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533312 00000100:00100000:3.0:1761315131.084530:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4021 00000100:00100000:3.0:1761315131.084548:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307533248:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315131.084559:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4022 00000100:00100000:0.0:1761315131.084566:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307533312:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315131.084602:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307533248:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 65us (469us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315131.084604:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307533312:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 36us (161us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315131.084616:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4021 00000100:00100000:0.0:1761315131.084875:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4022 00000100:00100000:0.0:1761315131.611840:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322479168 00000100:00100000:0.0:1761315131.611886:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2525 00000100:00100000:0.0:1761315131.611892:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322479168:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315131.611923:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315131.611965:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322479168:12345-0@lo:41:osp-pre-1-0.0 Request processed in 72us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315131.611972:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2525 00000100:00100000:0.0:1761315131.612015:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b898b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322479168:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315132.124803:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315134.173389:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322479936:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315134.173423:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315134.173467:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322479936 00000100:00100000:3.0:1761315134.173494:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1761315134.173500:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322479936:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315134.173525:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315134.173568:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322479936:12345-0@lo:41:osp-pre-0-1.0 Request processed in 66us (144us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315134.173575:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1761315134.173608:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322479936:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315134.173665:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322479232 00000100:00100000:3.0:1761315134.173695:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1510 00000100:00100000:3.0:1761315134.173701:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322479232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315134.173740:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322479232:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (813us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315134.173750:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1510 00000100:00100000:3.0:1761315134.173808:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322480000 00000100:00100000:3.0:1761315134.173839:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 905 00000100:00100000:3.0:1761315134.173845:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322480000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315134.173864:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315134.173897:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322480000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (123us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315134.173903:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 905 00000100:00080000:3.0:1761315134.174897:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315134.174903:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315134.174907:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011e09f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322479552:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315134.174915:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315134.174918:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315134.174920:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322479616:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315135.195532:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315136.199874:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315136.199925:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315136.199963:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315136.200241:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533504 00000100:00100000:3.0:1761315136.200278:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533568 00000100:00100000:0.0:1761315136.205472:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315136.733339:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322480064 00000100:00100000:3.0:1761315136.733974:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2529 00000100:00100000:3.0:1761315136.733981:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322480064:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315136.734016:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315136.734071:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322480064:12345-0@lo:41:osp-pre-1-0.0 Request processed in 89us (842us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315136.734078:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2529 00010000:00010000:0.0:1761315138.267498:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315139.291524:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315139.291546:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322480192:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315139.291612:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315139.291655:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322480128:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315139.291683:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315139.293791:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322480128 00000100:00100000:0.0:1761315139.293848:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2530 00000100:00100000:0.0:1761315139.293853:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f631184 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322480128:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315139.293886:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315139.293957:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f631184 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322480128:12345-0@lo:41:osp-pre-0-1.0 Request processed in 103us (2273us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315139.293963:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2530 00010000:00010000:0.0:1761315139.294326:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315139.297753:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322480448 00000100:00100000:3.0:1761315139.297783:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2531 00000100:00100000:3.0:1761315139.297786:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322480448:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315139.297824:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322480448:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (448us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315139.297829:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2531 00000100:00100000:0.0:1761315141.321955:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315141.322005:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315141.322054:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315141.322067:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533696 00000100:00100000:3.0:1761315141.322190:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1513 00000100:00100000:3.0:1761315141.322200:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307533696:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315141.322247:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533760 00000100:00100000:0.0:1761315141.322301:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4033 00000100:00100000:3.0:1761315141.322308:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307533696:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 105us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315141.322319:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071a6088f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307533760:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315141.322332:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1513 00000100:00100000:0.0:1761315141.322450:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071a6088f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307533760:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 138us (395us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315141.322459:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4033 00000100:00100000:3.0:1761315141.330296:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00010000:00010000:0.0:1761315141.339736:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315141.851561:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315141.851586:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322480832:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315141.851646:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315141.852024:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322480832:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315142.363517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315144.411838:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315144.411845:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315144.411848:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315144.411851:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322480896:0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315144.411856:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3386 00000100:00080000:0.0:1761315144.411861:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315144.411865:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315144.411886:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a8a536d5 x1846869322481216/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315144.411902:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315144.411918:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315144.411920:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315144.411923:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3386 00000100:00080000:0.0:1761315144.411928:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315144.411932:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315144.411936:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315144.411945:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007dab8c06 x1846869322481280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315144.411965:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481152:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315144.412009:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315144.412012:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322481024 00000100:00100000:0.0:1761315144.412046:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322481152 00000100:00100000:0.0:1761315144.412061:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2533 00000100:00100000:0.0:1761315144.412066:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322481024:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412103:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322481024:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (161us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315144.412121:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2533 00000100:00100000:0.0:1761315144.412135:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2534 00000100:00100000:0.0:1761315144.412138:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322481152:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412163:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322481152:12345-0@lo:400:kworker/u8:1.0 Request processed in 25us (156us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315144.412168:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2534 00000100:00100000:0.0:1761315144.412183:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322481216:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412209:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315144.412213:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481536:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412223:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dab8c06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322481280:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412262:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4036 00000100:00100000:0.0:1761315144.412266:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3abc72e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322480960:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315144.412289:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3abc72e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322480960:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (225us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315144.412557:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315144.414504:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4036 00000100:00100000:3.0:1761315144.414857:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481536:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315144.414870:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322480896:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315144.414880:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481152:0@lo:400:kworker/u8:1.0 00010000:00010000:0.0:1761315145.436698:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315146.440967:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534016 00000100:00100000:0.0:1761315146.441805:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307533952 00000100:00100000:0.0:1761315146.441840:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1515 00000100:00100000:0.0:1761315146.441845:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307533952:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315146.441875:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307533952:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 30us (1592us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315146.441881:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1515 00000100:00100000:3.0:1761315146.443781:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534080 00000100:00100000:3.0:1761315146.443820:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4040 00000100:00100000:3.0:1761315146.443826:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307534080:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315146.443883:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307534080:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 57us (456us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315146.443890:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4040 00000100:00100000:3.0:1761315146.972163:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315146.972392:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481728:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315146.972459:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315146.973218:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322481728:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315148.507710:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315149.531603:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322481792:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315149.531668:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315149.531707:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315149.531723:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322481792 00000100:00100000:3.0:1761315149.531770:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2537 00000100:00100000:3.0:1761315149.531776:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322481792:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315149.531809:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315149.531849:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322481792:12345-0@lo:41:osp-pre-0-1.0 Request processed in 71us (181us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.531910:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2537 00000100:00100000:3.0:1761315149.531938:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322482048:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.531973:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315149.531995:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b6ed247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322482432:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.532088:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009c0b045b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322481856:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.532641:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315149.532679:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322481792:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315149.532699:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315149.532706:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322481984:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.532795:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315149.532814:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322482368:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.532848:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482496 00000100:00100000:3.0:1761315149.533119:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 908 00000100:00100000:3.0:1761315149.533123:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322482496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315149.533141:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315149.533173:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322482496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (514us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.533180:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 908 00000100:00100000:3.0:1761315149.533204:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322481920 00000100:00100000:3.0:1761315149.533224:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482304 00000100:00100000:3.0:1761315149.533238:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322481984 00000100:00100000:3.0:1761315149.533250:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4041 00000100:00100000:3.0:1761315149.533253:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322481920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.533277:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322481920:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (1327us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.533282:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4041 00000100:00100000:3.0:1761315149.533289:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4042 00000100:00100000:3.0:1761315149.533292:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322482304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.533308:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322482304:12345-0@lo:400:kworker/u8:0.0 Request processed in 15us (1308us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.533312:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4042 00000100:00100000:3.0:1761315149.533318:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4043 00000100:00100000:3.0:1761315149.533320:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000239d651f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322481984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.533335:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000239d651f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322481984:12345-0@lo:400:kworker/u8:0.0 Request processed in 14us (543us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.533339:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4043 00000100:00100000:3.0:1761315149.533379:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315149.533399:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322481984:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.533435:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482368 00000100:00100000:3.0:1761315149.533451:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1359 00000100:00100000:3.0:1761315149.533454:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a70ec9ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322482368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.533899:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315149.533922:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322482240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534142:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315149.534147:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a70ec9ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322482368:12345-0@lo:400:kworker/u8:0.0 Request processed in 690us (766us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315149.534159:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1359 00000100:00100000:3.0:1761315149.534179:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1983 00000100:00100000:3.0:1761315149.534185:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322482176:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534189:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482240 00000100:00100000:0.0:1761315149.534228:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4044 00000100:00100000:0.0:1761315149.534233:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec256aee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322482240:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761315149.534270:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315149.534272:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec256aee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322482240:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (130us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315149.534280:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4044 00000100:00100000:3.0:1761315149.534314:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322482176:12345-0@lo:8:kworker/u8:0.0 Request processed in 127us (1479us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315149.534322:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1983 00000100:00100000:0.0:1761315149.534331:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322482240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534380:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009c0b045b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322481856:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534476:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322482496:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315149.534487:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322482368:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534499:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322481920:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.534505:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322482048:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315149.534517:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322482304:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315149.534521:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b6ed247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322482432:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315151.561579:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315151.561627:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315151.566569:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315151.566599:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534272 00000100:00100000:0.0:1761315151.566657:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4045 00000100:00100000:0.0:1761315151.566665:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfbf26b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307534272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315151.566762:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfbf26b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307534272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 97us (5135us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315151.566770:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4045 00000100:00100000:0.0:1761315151.568509:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534336 00000100:00100000:0.0:1761315151.568562:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4046 00000100:00100000:0.0:1761315151.568568:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000288e619e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307534336:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315151.568633:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000288e619e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307534336:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (2073us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315151.568640:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4046 00010000:00010000:0.0:1761315151.579738:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315152.092040:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322482560:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315152.092151:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315152.603602:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315154.651663:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315154.651671:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315154.651675:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315154.651680:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3397 00000100:00080000:0.0:1761315154.651683:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315154.651689:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315154.651710:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009e135009 x1846869322483072/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315154.651725:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315154.651729:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315154.651730:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315154.651739:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3397 00000100:00080000:0.0:1761315154.651747:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315154.651751:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315154.651763:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007dab8c06 x1846869322483136/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761315154.651862:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315154.651892:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482816 00000100:00100000:3.0:1761315154.651941:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a3ff52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322482688:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315154.651944:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483200 00000100:00100000:0.0:1761315154.651970:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4047 00000100:00100000:0.0:1761315154.651978:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322482816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315154.653322:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315154.653376:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322482944:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315154.653428:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315154.653463:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322483328:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315154.653512:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315154.653579:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482944 00000100:00100000:3.0:1761315154.653614:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4050 00000100:00100000:3.0:1761315154.653621:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000815d8bc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322482944:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315154.653685:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000815d8bc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322482944:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315154.653697:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4050 00000100:00100000:3.0:1761315154.653743:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322482688 00000100:00100000:3.0:1761315154.653793:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 909 00000100:00100000:3.0:1761315154.653801:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9b8c817 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322482688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315154.653839:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315154.653846:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322482816:12345-0@lo:400:kworker/u8:0.0 Request processed in 1862us (2090us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315154.653857:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4047 00000100:00100000:3.0:1761315154.653896:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9b8c817 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322482688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 90us (572us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315154.653907:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 909 00000100:00080000:0.0:1761315154.653964:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315154.653970:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315154.653975:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dab8c06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322483136:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315154.653981:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a3ff52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322482688:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315154.654014:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322482944:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315154.654105:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322483328:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315154.654474:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315154.654480:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315154.654484:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322483072:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315155.675737:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315156.681697:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315156.681761:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315156.682021:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315156.682318:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315156.683003:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534528 00000100:00100000:0.0:1761315156.683078:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534592 00000100:00100000:0.0:1761315156.683103:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4051 00000100:00100000:0.0:1761315156.683109:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f24932f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307534528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315156.683167:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f24932f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307534528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 59us (1484us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315156.683174:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4051 00000100:00100000:0.0:1761315156.683188:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4052 00000100:00100000:0.0:1761315156.683191:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004dec0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307534592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315156.683215:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004dec0a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307534592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (1194us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315156.683220:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4052 00000100:00100000:3.0:1761315157.212795:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322483456:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315157.212885:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315157.213800:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483456 00000100:00100000:0.0:1761315157.213864:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2543 00000100:00100000:0.0:1761315157.213870:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322483456:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315157.213914:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315157.213979:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322483456:12345-0@lo:41:osp-pre-1-0.0 Request processed in 108us (1096us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315157.213986:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2543 00010000:00010000:0.0:1761315157.723603:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315158.748427:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315159.771528:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315159.772193:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322483648:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772260:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315159.772296:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041655014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322484032:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772325:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315159.772367:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315159.772375:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e178d65c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322483712:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772411:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315159.772433:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008f170fef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322484096:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772464:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315159.772506:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483648 00000100:00100000:0.0:1761315159.772542:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484032 00000100:00100000:0.0:1761315159.772567:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484096 00000100:00100000:0.0:1761315159.772585:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4053 00000100:00100000:0.0:1761315159.772589:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322483648:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772633:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322483648:12345-0@lo:400:kworker/u8:1.0 Request processed in 43us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.772640:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4053 00000100:00100000:0.0:1761315159.772659:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4054 00000100:00100000:0.0:1761315159.772663:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dce6b44d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322484032:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772700:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dce6b44d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322484032:12345-0@lo:400:kworker/u8:1.0 Request processed in 34us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.772708:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4054 00000100:00100000:0.0:1761315159.772725:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4055 00000100:00100000:0.0:1761315159.772729:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000890fed6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322484096:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772753:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000890fed6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322484096:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.772758:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4055 00000100:00100000:0.0:1761315159.772799:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008f170fef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322484096:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772811:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041655014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322484032:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.772818:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322483648:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.773365:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483712 00000100:00100000:0.0:1761315159.773397:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483584 00000100:00100000:0.0:1761315159.773409:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483840 00000100:00100000:0.0:1761315159.773426:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2544 00000100:00100000:0.0:1761315159.773430:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322483712:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.773475:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322483712:12345-0@lo:400:kworker/u8:1.0 Request processed in 43us (1066us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.773482:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2544 00000100:00100000:3.0:1761315159.773497:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2545 00000100:00100000:3.0:1761315159.773510:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322483584:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315159.773519:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2546 00000100:00100000:0.0:1761315159.773527:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322483840:12345-0@lo:400:kworker/u8:1.0 00000004:00000024:3.0:1761315159.773561:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315159.773566:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322483840:12345-0@lo:400:kworker/u8:1.0 Request processed in 37us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.773574:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2546 00000100:00100000:0.0:1761315159.773611:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e178d65c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322483712:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315159.773645:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322483584:12345-0@lo:41:osp-pre-0-1.0 Request processed in 134us (383us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315159.773654:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2545 00000100:00100000:0.0:1761315159.773701:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484224 00000100:00100000:3.0:1761315159.773732:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484288 00000100:00100000:0.0:1761315159.773753:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1362 00000100:00100000:0.0:1761315159.773762:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322484224:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315159.773801:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 910 00000100:00100000:3.0:1761315159.773813:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322484288:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315159.773816:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322484224:12345-0@lo:400:kworker/u8:1.0 Request processed in 52us (475us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315159.773832:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1362 00002000:00000024:3.0:1761315159.773850:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315159.773939:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322484288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 121us (511us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315159.773968:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 910 00000100:00100000:0.0:1761315159.773973:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322484288:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315159.774074:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322483776 00000100:00100000:0.0:1761315159.774108:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4056 00000100:00100000:0.0:1761315159.774117:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322483776:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315159.774140:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484160 00000100:00100000:0.0:1761315159.774172:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322483776:12345-0@lo:400:kworker/u8:1.0 Request processed in 56us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315159.774193:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1363 00000100:00100000:3.0:1761315159.774200:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322484160:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315159.774202:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4056 00000100:00100000:3.0:1761315159.774262:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322484160:12345-0@lo:400:kworker/u8:1.0 Request processed in 61us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315159.774270:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1363 00010000:00010000:0.0:1761315160.796056:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315161.802588:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315161.802635:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315161.802655:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534912 00000100:00100000:0.0:1761315161.802699:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1990 00000100:00100000:0.0:1761315161.802708:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307534912:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315161.802786:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315161.802838:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307534912:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 130us (259us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315161.802844:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1990 00000100:00100000:3.0:1761315161.803602:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315161.803657:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00010000:00010000:0.0:1761315161.820123:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315162.331576:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322484352:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315162.331647:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315162.332002:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322484352:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315162.845745:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315163.867517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315164.891503:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315164.891530:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315164.891550:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322484480:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315164.891618:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315164.891658:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322484416:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315164.891930:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315164.891975:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322484544:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315164.892007:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315164.892014:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315164.892016:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315164.892021:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315164.892027:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3407 00000100:00080000:3.0:1761315164.892030:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315164.892037:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315164.892056:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ebd8a2b4 x1846869322484800/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315164.892058:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322484480:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315164.892078:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315164.892080:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:3.0:1761315164.892083:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315164.892085:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761315164.892089:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322484736:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315164.892091:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3407 00000100:00080000:3.0:1761315164.892095:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315164.892101:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315164.892117:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000421e79db x1846869322484864/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315164.892127:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315164.892153:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322484928:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315164.892189:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315164.892216:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484736 00000100:00100000:0.0:1761315164.892233:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322484544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892251:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2549 00000100:00100000:3.0:1761315164.892258:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322484736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315164.892260:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322484416:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315164.892297:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322485056:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892308:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322484736:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315164.892318:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2549 00000100:00100000:0.0:1761315164.892334:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315164.892366:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4060 00000100:00100000:3.0:1761315164.892373:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322484672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315164.892381:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315164.892390:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000da32678b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322484992:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892422:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322484672:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (324us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315164.892435:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315164.892440:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4060 00000100:00100000:3.0:1761315164.892464:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484992 00000100:00100000:0.0:1761315164.892491:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485056 00000100:00100000:3.0:1761315164.892495:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4062 00000100:00100000:3.0:1761315164.892502:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041655014 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322484992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315164.892531:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322484864 00000100:00100000:3.0:1761315164.892548:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041655014 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322484992:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (119us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315164.892559:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4062 00000100:00100000:0.0:1761315164.892567:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1991 00000100:00100000:0.0:1761315164.892575:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322484800:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892605:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322484928:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892626:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322484736:0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761315164.892686:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315164.892688:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa2b8875 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322484672:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315164.892699:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322484608:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315164.894995:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322484800:12345-0@lo:8:kworker/u8:0.0 Request processed in 2419us (2602us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315164.895007:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1991 00000100:00100000:0.0:1761315166.923187:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307534976 00000100:00100000:0.0:1761315166.923246:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1523 00000100:00100000:0.0:1761315166.923252:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307534976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315166.923322:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307534976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 68us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315166.923330:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1523 00000100:00100000:0.0:1761315166.923406:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4063 00000100:00100000:0.0:1761315166.923412:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000932fbacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307535040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315166.923448:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000932fbacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307535040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315166.923455:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4063 00000100:00100000:3.0:1761315166.924317:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315166.924780:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535168 00000100:00100000:0.0:1761315166.924909:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1993 00000100:00100000:0.0:1761315166.924914:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307535168:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315166.925132:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315166.925191:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307535168:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 275us (875us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315166.925199:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1993 00010000:00010000:3.0:1761315166.939486:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315167.451569:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322485120:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315167.451643:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315167.451712:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485120 00000100:00100000:3.0:1761315167.452328:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2550 00000100:00100000:3.0:1761315167.452336:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f170fef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322485120:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315167.452403:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315167.453767:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f170fef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322485120:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1424us (2119us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315167.453780:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2550 00000100:00100000:0.0:1761315167.453873:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322485120:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315167.963654:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315170.011534:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:3.0:1761315170.011551:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315170.011557:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761315170.011558:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322485376:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315170.011562:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315170.011570:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3412 00000100:00080000:3.0:1761315170.011573:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315170.011578:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315170.011599:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000007c7b302 x1846869322485504/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315170.011611:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315170.011615:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315170.011616:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315170.011619:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3412 00000100:00080000:3.0:1761315170.011621:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315170.011623:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315170.011633:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315170.011638:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f1ae97ad x1846869322485568/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315170.011673:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322485184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.011707:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761315170.011733:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315170.011746:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322485248:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.011752:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ebd8a2b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322485760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.011795:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315170.011800:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315170.012717:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000dc5d609 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322485632:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.012761:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485376 00000100:00100000:0.0:1761315170.012822:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485248 00000100:00100000:0.0:1761315170.012852:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485696 00000100:00100000:0.0:1761315170.012886:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4065 00000100:00100000:3.0:1761315170.012890:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315170.012896:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322485376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.012912:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000849700d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322485888:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315170.012963:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315170.012968:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322485376:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (1336us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315170.012978:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4065 00000100:00100000:0.0:1761315170.013000:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485632 00000100:00100000:3.0:1761315170.013017:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4066 00000100:00100000:3.0:1761315170.013024:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322485248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013034:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4067 00000100:00100000:0.0:1761315170.013040:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a3ff52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322485696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.013072:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322485248:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (1280us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315170.013082:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4066 00000100:00100000:3.0:1761315170.013105:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4068 00000100:00100000:3.0:1761315170.013111:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322485632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013192:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a3ff52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322485696:12345-0@lo:400:kworker/u8:0.0 Request processed in 150us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315170.013202:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4067 00000100:00100000:0.0:1761315170.013236:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1366 00000100:00100000:0.0:1761315170.013244:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322485824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013288:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322485824:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (557us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315170.013296:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1366 00000100:00100000:0.0:1761315170.013318:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1995 00000100:00100000:0.0:1761315170.013326:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322485568:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315170.013374:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322485632:12345-0@lo:400:kworker/u8:0.0 Request processed in 112us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315170.013387:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4068 00000100:00100000:3.0:1761315170.013414:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322485888 00000100:00100000:3.0:1761315170.013448:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 912 00000100:00100000:3.0:1761315170.013454:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322485888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315170.013473:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315170.013507:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322485888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (546us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315170.013514:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 912 00000100:00100000:3.0:1761315170.013569:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322485248:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.013595:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000dc5d609 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322485632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.013616:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000849700d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322485888:0@lo:13:osp-pre-1-1.0 00010000:02020000:0.0:1761315170.013695:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315170.013788:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322485568:12345-0@lo:8:kworker/u8:0.0 Request processed in 463us (800us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315170.013796:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1995 00000100:00100000:3.0:1761315170.013810:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ebd8a2b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322485760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315170.013824:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322485376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013842:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322485312:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013852:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000421e79db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322485696:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315170.013861:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322485952:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315170.014786:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322485184:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315172.040138:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315172.040193:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315172.040215:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535296 00000100:00100000:0.0:1761315172.040291:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4069 00000100:00100000:0.0:1761315172.040298:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307535296:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315172.040400:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307535296:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 101us (269us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315172.040408:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4069 00000100:00100000:3.0:1761315172.041458:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315172.043929:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315172.043984:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535424 00000100:00100000:0.0:1761315172.044032:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1996 00000100:00100000:0.0:1761315172.044038:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307535424:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315172.044126:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315172.044191:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307535424:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 151us (275us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315172.044198:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1996 00000100:00100000:3.0:1761315172.573312:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322486016:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315172.573407:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315172.573743:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322486016:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315173.085876:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315175.131568:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315175.131576:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315175.131580:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315175.131587:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3417 00000100:00080000:3.0:1761315175.131591:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315175.131597:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315175.131616:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000095732ed x1846869322486400/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315175.131632:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315175.131635:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315175.131636:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315175.131638:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3417 00000100:00080000:3.0:1761315175.131640:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315175.131642:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315175.131652:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bf5b9d47 x1846869322486464/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315175.131698:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322486144:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315175.132321:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000793073e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322486208:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315175.132373:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315175.132704:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315175.132722:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005b1075d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322486528:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315175.132847:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315175.132868:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322486784:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315175.132888:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315175.132930:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486784 00000100:00100000:3.0:1761315175.132955:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 913 00000100:00100000:3.0:1761315175.132959:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322486784:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315175.133425:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322486592:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315175.133459:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315175.133574:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486272 00000100:00100000:0.0:1761315175.133606:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486144 00000100:00100000:0.0:1761315175.133628:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486528 00002000:00000024:3.0:1761315175.133651:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315175.133690:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322486784:12345-0@lo:13:osp-pre-1-1.0 Request processed in 728us (800us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315175.133698:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 913 00000100:00100000:2.0:1761315175.133887:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322486272:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (2238us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315175.133904:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4071 00000100:00100000:2.0:1761315175.133919:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4072 00000100:00100000:2.0:1761315175.133924:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322486144:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315175.133959:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322486144:12345-0@lo:400:kworker/u8:1.0 Request processed in 32us (1253us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315175.133967:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4072 00000100:00100000:2.0:1761315175.133983:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4073 00000100:00100000:2.0:1761315175.133987:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322486528:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315175.134011:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322486528:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (1163us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315175.134018:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4073 00000100:00100000:2.0:1761315175.134030:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4074 00000100:00100000:2.0:1761315175.134035:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322486592:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315175.134071:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322486592:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (614us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315175.134077:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4074 00010000:00010000:3.0:1761315176.157187:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315177.159775:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315177.159814:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315177.159845:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315177.159902:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535616 00000100:00100000:3.0:1761315177.159902:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535552 00000100:00100000:3.0:1761315177.159954:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4075 00000100:00100000:3.0:1761315177.159960:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005b1075d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307535552:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315177.159968:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4076 00000100:00100000:0.0:1761315177.159975:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307535616:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315177.160023:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005b1075d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307535552:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 61us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315177.160025:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307535616:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 48us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315177.160036:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4075 00000100:00100000:0.0:1761315177.160042:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4076 00000100:00100000:3.0:1761315177.160456:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315177.160572:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535680 00000100:00100000:0.0:1761315177.160599:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1999 00000100:00100000:0.0:1761315177.160603:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307535680:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315177.160688:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315177.160780:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307535680:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 176us (325us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315177.160787:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1999 00000100:00100000:2.0:1761315177.161948:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535488 00000100:00100000:2.0:1761315177.162005:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1527 00000100:00100000:2.0:1761315177.162010:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307535488:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315177.162067:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307535488:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 56us (2300us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315177.162074:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1527 00010000:00010000:3.0:1761315177.180743:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315177.691548:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322486912:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315177.692188:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315177.693300:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486912 00000100:00100000:0.0:1761315177.693358:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2558 00000100:00100000:0.0:1761315177.693364:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322486912:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315177.693400:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315177.693811:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322486912:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315177.693899:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322486912:12345-0@lo:41:osp-pre-1-0.0 Request processed in 534us (1713us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315177.693907:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2558 00010000:00010000:3.0:1761315179.227580:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315180.251912:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315180.251918:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315180.251921:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315180.251927:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3422 00000100:00080000:2.0:1761315180.251929:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315180.251935:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315180.251954:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d28148bd x1846869322487424/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315180.251968:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315180.251972:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322487104:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315180.251973:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315180.251976:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315180.251982:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3422 00000100:00080000:2.0:1761315180.251984:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315180.251987:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315180.252000:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023426d14 x1846869322487488/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315180.252013:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315180.252035:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2dfaf30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322487360:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315180.252045:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322487168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.252063:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315180.252106:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487360 00000100:00100000:2.0:1761315180.252149:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2559 00000100:00100000:2.0:1761315180.252155:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007fe660ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322487360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315180.252240:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322486976 00000100:00100000:3.0:1761315180.252267:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 914 00000100:00100000:3.0:1761315180.252271:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322486976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315180.252290:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315180.252908:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007fe660ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322487360:12345-0@lo:400:kworker/u8:0.0 Request processed in 751us (845us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315180.252921:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2559 00000100:00100000:2.0:1761315180.255817:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322487232:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315180.255910:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315180.255950:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322487616:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315180.256107:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322486976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3833us (4368us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315180.256118:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 914 00000100:00100000:3.0:1761315180.256165:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487104 00000100:00100000:3.0:1761315180.256206:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4077 00000100:00100000:3.0:1761315180.256211:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322487104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315180.256267:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322487104:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (4248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315180.256274:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4077 00010000:00010000:3.0:1761315180.256316:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315180.256334:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487232 00000100:00100000:3.0:1761315180.256379:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2560 00000100:00100000:3.0:1761315180.256383:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322487232:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315180.256819:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315180.256994:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2dfaf30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322487360:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.257034:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322487104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.257063:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315180.257088:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322487552:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.257213:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487424 00000100:00100000:0.0:1761315180.257239:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487488 00000100:00100000:0.0:1761315180.257259:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2000 00000100:00100000:0.0:1761315180.257264:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322487424:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315180.257504:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315180.257555:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322487424:12345-0@lo:8:kworker/u8:0.0 Request processed in 289us (556us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315180.257564:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2000 00000100:00100000:2.0:1761315180.257876:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315180.257928:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315180.257958:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e178d65c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322486976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315180.260035:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487616 00000100:00100000:0.0:1761315180.260096:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487552 00000100:00100000:0.0:1761315180.260119:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4078 00000100:00100000:0.0:1761315180.260125:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000359981a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322487616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.260214:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000359981a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322487616:12345-0@lo:400:kworker/u8:0.0 Request processed in 89us (2338us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315180.260220:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4078 00000100:00100000:0.0:1761315180.260234:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4079 00000100:00100000:0.0:1761315180.260237:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004564b6ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322487552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.260259:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004564b6ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322487552:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (2333us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315180.260264:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4079 00000100:00100000:0.0:1761315180.260370:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322487616:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315180.260746:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322487168:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315180.260757:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322487552:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.261015:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487296 00000100:00100000:0.0:1761315180.261043:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4080 00000100:00100000:0.0:1761315180.261047:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dab8c06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322487296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315180.261080:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dab8c06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322487296:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (208us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315180.261086:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4080 00000100:00100000:2.0:1761315180.261946:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487680 00000100:00100000:2.0:1761315180.261984:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1369 00000100:00100000:2.0:1761315180.261988:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322487680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315180.262038:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322487680:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (1095us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315180.262045:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1369 00000100:00100000:0.0:1761315180.262109:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322487680:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315180.262211:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322487232:12345-0@lo:41:osp-pre-0-1.0 Request processed in 5827us (6302us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315180.262219:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2560 00000100:00100000:0.0:1761315180.262281:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322487232:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315182.280783:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315182.280866:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315182.281093:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535808 00000100:00100000:2.0:1761315182.282332:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535744 00000100:00100000:3.0:1761315182.282379:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315182.282601:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315182.282640:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535936 00000100:00100000:3.0:1761315182.282673:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2002 00000100:00100000:3.0:1761315182.282682:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307535936:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315182.282707:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1529 00000100:00100000:2.0:1761315182.282714:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307535744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315182.282811:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307535744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 97us (1945us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315182.282821:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1529 00000100:00100000:2.0:1761315182.283114:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307535872 00000100:00100000:2.0:1761315182.283146:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4081 00000100:00100000:2.0:1761315182.283153:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307535808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315182.283195:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307535808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (2430us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315182.283202:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4081 00000100:00100000:2.0:1761315182.283215:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4082 00000100:00100000:2.0:1761315182.283218:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307535872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315182.283241:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307535872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 22us (868us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315182.283246:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4082 00010000:02020000:3.0:1761315182.283616:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315182.283658:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307535936:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 976us (1058us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315182.283681:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2002 00010000:00010000:3.0:1761315182.301208:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315182.811731:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315182.811768:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322487744:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315182.811883:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315182.812046:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487744 00000100:00100000:3.0:1761315182.812887:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2562 00000100:00100000:3.0:1761315182.812899:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322487744:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315182.812949:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315182.813038:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322487744:12345-0@lo:41:osp-pre-1-0.0 Request processed in 139us (1156us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315182.813047:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2562 00000100:00100000:2.0:1761315182.813103:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322487744:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315183.324094:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315185.371683:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000883ca932 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322487808:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761315185.371685:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315185.371754:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322487936:0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315185.371764:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315185.371771:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761315185.371775:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315185.371777:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315185.371785:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3427 00000100:00080000:0.0:1761315185.371788:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315185.371796:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315185.371799:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.371841:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001ca88bcf x1846869322488192/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315185.371853:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1761315185.371856:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315185.371859:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322488000:0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315185.371860:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315185.371861:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315185.371865:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3427 00000100:00080000:0.0:1761315185.371866:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315185.371869:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315185.371880:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e178d65c x1846869322488256/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315185.371881:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315185.371898:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322488320:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.371928:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322488384:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315185.371934:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.371956:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.372004:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487808 00000100:00100000:2.0:1761315185.372013:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322488064:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372061:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.372062:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488000 00000100:00100000:0.0:1761315185.372095:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2563 00000100:00100000:3.0:1761315185.372098:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487936 00000100:00100000:2.0:1761315185.372104:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2dfaf30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322488448:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.372105:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322487808:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315185.372133:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488320 00000100:00100000:2.0:1761315185.372148:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:0.0:1761315185.372155:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315185.372167:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488384 00000100:00100000:3.0:1761315185.372185:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488064 00000100:00100000:3.0:1761315185.372204:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4083 00000100:00100000:3.0:1761315185.372208:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322487936:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372227:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4084 00000100:00100000:2.0:1761315185.372234:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322488320:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.372240:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322487808:12345-0@lo:41:osp-pre-0-1.0 Request processed in 136us (468us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315185.372263:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2563 00000100:00100000:3.0:1761315185.372267:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322487936:12345-0@lo:400:kworker/u8:1.0 Request processed in 59us (470us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315185.372280:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4083 00000100:00100000:0.0:1761315185.372290:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2564 00000100:00100000:0.0:1761315185.372298:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322488000:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372301:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322488320:12345-0@lo:400:kworker/u8:1.0 Request processed in 63us (362us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315185.372306:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4085 00000100:00100000:3.0:1761315185.372315:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005167c36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322488384:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372324:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4084 00000100:00100000:0.0:1761315185.372377:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322488000:12345-0@lo:400:kworker/u8:1.0 Request processed in 77us (493us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315185.372381:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4086 00000100:00100000:3.0:1761315185.372396:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005167c36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322488384:12345-0@lo:400:kworker/u8:1.0 Request processed in 72us (430us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315185.372400:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2564 00000100:00100000:2.0:1761315185.372400:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322488064:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315185.372408:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4085 00000100:00100000:3.0:1761315185.372431:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322488576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315185.372450:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322487872 00000100:00100000:2.0:1761315185.372453:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322488064:12345-0@lo:400:kworker/u8:1.0 Request processed in 61us (392us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315185.372462:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4086 00000100:00100000:3.0:1761315185.372468:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.372484:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1530 00000100:00100000:0.0:1761315185.372488:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322487872:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372509:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488448 00000100:00100000:3.0:1761315185.372510:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322488320:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.372519:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322487872:12345-0@lo:400:kworker/u8:1.0 Request processed in 31us (305us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315185.372523:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322488000:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.372526:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1530 00000100:00100000:3.0:1761315185.372535:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322487936:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372550:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1370 00000100:00100000:2.0:1761315185.372557:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322488448:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.372572:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322488384:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315185.372572:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488576 00000100:00100000:0.0:1761315185.372599:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322488064:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315185.372602:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 915 00000100:00100000:3.0:1761315185.372607:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322488576:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315185.372611:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322488448:12345-0@lo:400:kworker/u8:1.0 Request processed in 52us (463us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315185.372616:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2dfaf30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322488448:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372624:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1370 00002000:00000024:3.0:1761315185.372632:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315185.372634:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000883ca932 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322487808:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315185.372672:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322488576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315185.372678:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 915 00000100:00100000:0.0:1761315185.372686:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322488192:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315185.372712:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322488576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315185.372715:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315185.372728:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e178d65c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322488256:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315185.372735:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488192 00000100:00100000:0.0:1761315185.372754:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315185.372773:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2003 00000100:00100000:2.0:1761315185.372777:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322488192:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315185.372900:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315185.372932:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322488192:12345-0@lo:8:kworker/u8:1.0 Request processed in 154us (218us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315185.372939:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2003 00000100:00100000:3.0:1761315185.373320:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488256 00000100:00080000:0.0:1761315185.373333:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315185.373338:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315185.373364:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2004 00000100:00100000:0.0:1761315185.373367:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322488192:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315185.373370:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322488256:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:3.0:1761315185.373449:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315185.373898:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488128 00000100:00100000:2.0:1761315185.373950:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2565 00000100:00100000:2.0:1761315185.373956:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322488128:12345-0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315185.374884:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315185.374891:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315185.374896:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e178d65c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322488256:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315185.375318:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322488256:12345-0@lo:8:kworker/u8:1.0 Request processed in 1947us (2563us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315185.375329:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2004 00000100:00100000:2.0:1761315185.376037:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322488128:12345-0@lo:400:kworker/u8:1.0 Request processed in 2079us (3699us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315185.376050:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2565 00000100:00100000:0.0:1761315185.376113:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322488512:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315185.376139:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322488128:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315187.405554:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315187.405590:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315187.405825:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315187.407247:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315187.407305:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536000 00000100:00100000:0.0:1761315187.407334:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536128 00000100:00100000:2.0:1761315187.407373:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1531 00000100:00100000:2.0:1761315187.407381:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307536000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315187.407448:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307536000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (1626us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315187.407455:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1531 00000100:00100000:2.0:1761315187.407492:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4087 00000100:00100000:2.0:1761315187.407497:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ef2c853 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307536064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315187.407530:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ef2c853 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307536064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (1983us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315187.407537:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4087 00000100:00100000:0.0:1761315187.407778:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4088 00000100:00100000:0.0:1761315187.407787:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307536128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315187.407796:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536192 00000100:00100000:3.0:1761315187.407848:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2005 00000100:00100000:3.0:1761315187.407855:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307536192:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:0.0:1761315187.407860:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307536128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 55us (598us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315187.407870:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4088 00010000:02020000:3.0:1761315187.407945:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315187.407980:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307536192:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 125us (2390us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315187.407987:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2005 00000100:00100000:0.0:1761315187.931609:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322488640:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315187.931678:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315187.931791:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488640 00000100:00100000:2.0:1761315187.931853:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2566 00000100:00100000:2.0:1761315187.931859:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322488640:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315187.931893:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315187.931948:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322488640:12345-0@lo:41:osp-pre-1-0.0 Request processed in 89us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315187.931956:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2566 00010000:00010000:3.0:1761315188.443649:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315190.491732:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315190.491751:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efdaddff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322488704:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315190.491846:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315190.492107:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322489088:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315190.493729:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315190.493770:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322489152:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315190.493808:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315190.493860:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489088 00000100:00100000:2.0:1761315190.493899:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489152 00000100:00100000:3.0:1761315190.493917:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322488960:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.493923:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2006 00000100:00100000:2.0:1761315190.493931:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489088:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315190.493943:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322488768:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315190.493977:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315190.493994:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315190.494009:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322489344:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.494024:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322489024:0@lo:400:kworker/u8:1.0 00010000:02020000:2.0:1761315190.494028:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315190.494053:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315190.494076:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315190.494102:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489088:12345-0@lo:8:kworker/u8:1.0 Request processed in 166us (373us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315190.494114:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2006 00000100:00100000:0.0:1761315190.494116:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322489408:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.494145:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2007 00000100:00100000:0.0:1761315190.494149:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315190.494152:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489152:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315190.494185:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489408 00000100:00100000:0.0:1761315190.494209:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1372 00000100:00100000:0.0:1761315190.494213:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322489408:12345-0@lo:400:kworker/u8:1.0 00010000:02020000:2.0:1761315190.494229:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315190.494265:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322489408:12345-0@lo:400:kworker/u8:1.0 Request processed in 50us (117us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.494269:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489152:12345-0@lo:8:kworker/u8:1.0 Request processed in 114us (458us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315190.494277:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1372 00000100:00100000:2.0:1761315190.494292:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2007 00000100:00100000:0.0:1761315190.494305:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488768 00000100:00100000:0.0:1761315190.494329:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1532 00000100:00100000:0.0:1761315190.494333:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322488768:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.494399:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322488768:12345-0@lo:400:kworker/u8:1.0 Request processed in 63us (406us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315190.494407:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1532 00000100:00100000:2.0:1761315190.494439:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489280 00000100:00100000:0.0:1761315190.494454:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489024 00000100:00100000:2.0:1761315190.494492:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488832 00000100:00100000:2.0:1761315190.494516:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489216 00000100:00100000:2.0:1761315190.494525:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322488960 00000100:00100000:2.0:1761315190.494715:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4089 00000100:00100000:2.0:1761315190.494721:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322489280:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.494776:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322489280:12345-0@lo:400:kworker/u8:1.0 Request processed in 54us (1270us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.494792:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4089 00000100:00100000:2.0:1761315190.494811:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4090 00000100:00100000:3.0:1761315190.494815:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489344 00000100:00100000:2.0:1761315190.494817:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322488832:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.494861:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322488832:12345-0@lo:400:kworker/u8:1.0 Request processed in 44us (1302us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.494867:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4090 00000100:00100000:3.0:1761315190.494873:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1373 00000100:00100000:3.0:1761315190.494880:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322489344:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.494887:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4091 00000100:00100000:2.0:1761315190.494894:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322489216:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315190.494951:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322489344:12345-0@lo:400:kworker/u8:1.0 Request processed in 68us (898us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.494957:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322489216:12345-0@lo:400:kworker/u8:1.0 Request processed in 57us (1364us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315190.494966:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1373 00000100:00100000:2.0:1761315190.494971:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4091 00000100:00100000:2.0:1761315190.494994:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4092 00000100:00100000:2.0:1761315190.495003:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322488960:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.495043:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322488960:12345-0@lo:400:kworker/u8:1.0 Request processed in 40us (1066us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.495052:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4092 00000100:00100000:3.0:1761315190.495085:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322488960:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.495115:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322488832:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.495251:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2568 00000100:00100000:0.0:1761315190.495258:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322488896:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315190.495518:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322489344:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315190.495574:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322489472:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315190.495609:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315190.495646:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489472 00000100:00100000:3.0:1761315190.495813:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 916 00000100:00100000:3.0:1761315190.495820:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322489472:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315190.495837:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322489216:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315190.495872:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322488896:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.495875:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322488896:12345-0@lo:400:kworker/u8:1.0 Request processed in 600us (2453us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315190.495901:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322489280:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.495908:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2568 00000100:00100000:0.0:1761315190.495971:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322488768:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315190.495997:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322489024:0@lo:400:kworker/u8:1.0 00000100:00080000:2.0:1761315190.495997:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315190.496004:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315190.496013:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322489088:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315190.496022:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322489408:0@lo:400:kworker/u8:1.0 00000100:00080000:2.0:1761315190.496030:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315190.496034:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315190.496038:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322489152:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315190.496047:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efdaddff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322488704:0@lo:41:osp-pre-0-1.0 00002000:00000024:3.0:1761315190.501372:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315190.501491:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322489472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 5669us (5879us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315190.501507:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 916 00000100:00100000:0.0:1761315190.505445:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322489472:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1761315191.515698:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315192.524852:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315192.524919:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315192.524966:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315192.525061:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315192.525815:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536448 00000100:00100000:0.0:1761315192.525844:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536256 00000100:00100000:2.0:1761315192.525865:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2008 00000100:00100000:2.0:1761315192.525871:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307536448:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:0.0:1761315192.525878:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1533 00000100:00100000:0.0:1761315192.525883:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307536256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315192.525909:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307536256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (998us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315192.525914:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1533 00010000:02020000:2.0:1761315192.525978:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315192.526043:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307536448:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 169us (1203us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315192.526050:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2008 00000100:00100000:2.0:1761315193.051578:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322489536:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315193.051645:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315193.053608:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322489536:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315195.611565:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011072150 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322489600:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315195.611595:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315195.611601:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315195.611604:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315195.611611:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3438 00000100:00080000:2.0:1761315195.611614:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315195.611620:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315195.611641:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315195.611643:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023426d14 x1846869322489920/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315195.611661:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315195.611665:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315195.611666:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315195.611670:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3438 00000100:00080000:2.0:1761315195.611673:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315195.611673:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322489856:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315195.611679:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315195.611695:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b86079b3 x1846869322489984/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315195.611706:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315195.611738:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322490240:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.611766:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322489920:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315195.611781:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315195.611866:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315195.611878:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322489664:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.612020:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315195.612038:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489856 00000100:00100000:2.0:1761315195.612046:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b86079b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322489984:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315195.612067:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2571 00000100:00100000:3.0:1761315195.612073:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322489856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.612083:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315195.612087:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315195.612117:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322490048:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.612132:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490240 00000100:00100000:3.0:1761315195.612421:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322489856:12345-0@lo:400:kworker/u8:0.0 Request processed in 347us (717us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315195.612429:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2571 00000100:00100000:3.0:1761315195.612453:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008f353c47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322489792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.613040:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489920 00000100:00100000:0.0:1761315195.613047:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315195.613055:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489984 00000100:00100000:2.0:1761315195.613067:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1374 00000100:00100000:2.0:1761315195.613072:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322490240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315195.613102:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efdaddff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322490304:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315195.613111:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322490240:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (1332us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.613117:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1374 00000100:00100000:2.0:1761315195.613128:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2009 00000100:00100000:2.0:1761315195.613131:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489920:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315195.613201:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:2.0:1761315195.613233:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315195.613269:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322489920:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (1250us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315195.613277:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2010 00000100:00100000:0.0:1761315195.613282:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322489984:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315195.613425:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315195.613432:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315195.613459:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322490176:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315195.613486:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322489984:12345-0@lo:8:kworker/u8:0.0 Request processed in 201us (1407us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315195.613503:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315195.613528:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:3.0:1761315195.613538:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060c89ebd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322489728:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315195.613567:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315195.613676:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2009 00000100:00100000:0.0:1761315195.613682:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2010 00000100:00100000:2.0:1761315195.613692:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490176 00000100:00100000:0.0:1761315195.613713:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1375 00000100:00100000:3.0:1761315195.613718:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322490112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315195.613721:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322490176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.613728:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322489728 00000100:00100000:3.0:1761315195.613763:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315195.613770:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2572 00000100:00100000:2.0:1761315195.613776:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322489728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315195.613779:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322490368:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315195.613813:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322490176:12345-0@lo:400:kworker/u8:0.0 Request processed in 77us (299us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.614306:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322489728:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.614314:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2572 00000100:00100000:2.0:1761315195.614337:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490112 00000100:00100000:2.0:1761315195.614371:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4097 00000100:00100000:2.0:1761315195.614375:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322489792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614404:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322489792:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (975us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.614411:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4097 00000100:00100000:2.0:1761315195.614422:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4098 00000100:00100000:2.0:1761315195.614426:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322490112:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614451:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322490112:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (690us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.614456:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4098 00000100:00100000:3.0:1761315195.614492:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315195.614500:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315195.614505:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315195.614511:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322489920:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315195.614526:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315195.614527:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008f353c47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322489792:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315195.614530:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315195.614537:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b86079b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322489984:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315195.614547:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322490176:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614569:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490368 00000100:00100000:3.0:1761315195.614582:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060c89ebd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322489728:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614595:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2573 00000100:00100000:3.0:1761315195.614598:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322490112:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614602:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322490368:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315195.614627:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315195.614644:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322490240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315195.614659:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322489856:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614666:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322490368:12345-0@lo:41:osp-pre-0-1.0 Request processed in 64us (843us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315195.614672:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011072150 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322489600:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614684:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2573 00000100:00100000:0.0:1761315195.614695:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1375 00000100:00100000:3.0:1761315195.614725:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322490368:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315195.614746:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490304 00000100:00100000:2.0:1761315195.614789:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 917 00000100:00100000:3.0:1761315195.614792:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322489664:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315195.614797:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322490304:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315195.614808:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322490048:0@lo:400:kworker/u8:0.0 00002000:00000024:2.0:1761315195.614816:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315195.614850:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322490304:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (1648us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315195.614856:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 917 00000100:00100000:3.0:1761315195.614882:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efdaddff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322490304:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315197.640493:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315197.641387:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315197.642321:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315197.642370:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315197.642661:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536704 00000100:00100000:0.0:1761315197.642690:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2011 00000100:00100000:0.0:1761315197.642697:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307536704:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315197.642803:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315197.642856:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307536704:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 158us (1472us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315197.642863:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2011 00000100:00100000:0.0:1761315197.645434:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536512 00000100:00100000:0.0:1761315197.645482:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1535 00000100:00100000:0.0:1761315197.645488:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307536512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315197.645546:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307536512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 57us (3228us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315197.645553:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1535 00000100:00100000:2.0:1761315197.646390:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536640 00000100:00100000:2.0:1761315197.646426:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4099 00000100:00100000:2.0:1761315197.646432:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e178d65c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307536576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315197.646466:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e178d65c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307536576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (5981us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315197.646472:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4099 00000100:00100000:2.0:1761315197.646484:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4100 00000100:00100000:2.0:1761315197.646488:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307536640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315197.646505:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307536640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 17us (4137us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315197.646511:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4100 00000100:00100000:2.0:1761315198.171598:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315198.172816:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a08ec369 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322490432:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315198.172889:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315198.173071:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490432 00000100:00100000:3.0:1761315198.173269:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2574 00000100:00100000:3.0:1761315198.173276:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322490432:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315198.173307:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315198.173367:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322490432:12345-0@lo:41:osp-pre-1-0.0 Request processed in 91us (479us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315198.173375:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2574 00000100:00100000:2.0:1761315198.173488:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a08ec369 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322490432:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315198.683784:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315200.731487:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315200.731494:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315200.731497:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315200.731506:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3443 00000100:00080000:2.0:1761315200.731509:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315200.731514:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315200.731545:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000023426d14 x1846869322490816/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315200.731562:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315200.731565:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315200.731566:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315200.731569:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3443 00000100:00080000:2.0:1761315200.731571:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315200.731574:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315200.731586:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ec99d931 x1846869322490880/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315200.731634:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322490816:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315200.731696:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315200.731732:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322490880:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315200.731753:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315200.731904:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322490624:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.731936:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315200.731949:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322491008:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.731968:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315200.731982:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315200.731986:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322490688:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732001:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315200.732012:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322491072:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732027:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315200.732053:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491008 00000100:00100000:0.0:1761315200.732087:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490688 00000100:00100000:0.0:1761315200.732101:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4101 00000100:00100000:0.0:1761315200.732107:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322491008:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732152:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322491008:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (183us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315200.732160:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4101 00000100:00100000:0.0:1761315200.732175:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4102 00000100:00100000:0.0:1761315200.732178:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322490688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.732195:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490816 00000100:00100000:0.0:1761315200.732199:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322490688:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315200.732205:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4102 00000100:00100000:3.0:1761315200.732220:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490880 00000100:00100000:3.0:1761315200.732236:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491072 00000100:00100000:3.0:1761315200.732261:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1376 00000100:00100000:3.0:1761315200.732266:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322491072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732269:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010d50b50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322491264:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315200.732297:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315200.732309:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322491072:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (281us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315200.732317:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1376 00000100:00100000:0.0:1761315200.732330:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322490688:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.732331:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2012 00000100:00100000:3.0:1761315200.732339:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322490816:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732365:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322491072:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732374:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322491008:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732400:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490624 00000100:00100000:0.0:1761315200.732417:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491264 00000100:00100000:0.0:1761315200.732430:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2575 00000100:00100000:0.0:1761315200.732433:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322490624:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761315200.732450:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315200.732460:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322490624:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (524us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315200.732466:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2575 00000100:00100000:0.0:1761315200.732477:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2576 00000100:00100000:0.0:1761315200.732481:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a79f4c04 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322491264:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315200.732489:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322490816:12345-0@lo:8:kworker/u8:0.0 Request processed in 149us (792us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315200.732496:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2012 00000004:00000024:0.0:1761315200.732501:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315200.732511:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2013 00000100:00100000:3.0:1761315200.732516:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322490880:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315200.732589:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a79f4c04 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322491264:12345-0@lo:41:osp-pre-0-1.0 Request processed in 107us (293us total) trans 0 rc 0/0 00010000:02020000:3.0:1761315200.732595:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315200.732599:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2576 00000100:00100000:0.0:1761315200.732631:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010d50b50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322491264:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315200.732635:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322490880:12345-0@lo:8:kworker/u8:0.0 Request processed in 115us (878us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315200.732644:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322490624:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.732645:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2013 00000100:00100000:3.0:1761315200.732691:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322490560:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315200.732703:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315200.732737:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315200.732768:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322490944:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.732792:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315200.732811:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dfe7af88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491200:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315200.732847:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315200.732891:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490560 00000100:00100000:3.0:1761315200.732918:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490944 00000100:00100000:3.0:1761315200.732940:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4103 00000100:00100000:3.0:1761315200.732945:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efdaddff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322490560:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.732985:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efdaddff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322490560:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315200.732998:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4103 00000100:00100000:3.0:1761315200.733012:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4104 00000100:00100000:3.0:1761315200.733017:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322490944:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.733057:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322490944:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315200.733064:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4104 00000100:00100000:3.0:1761315200.733105:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322490560:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.733119:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322490944:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.733170:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322490752 00000100:00100000:3.0:1761315200.733199:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2577 00000100:00100000:3.0:1761315200.733203:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322490752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315200.733249:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322490752:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (358us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315200.733256:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2577 00000100:00100000:2.0:1761315200.733274:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491200 00000100:00080000:3.0:1761315200.733298:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315200.733302:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 918 00000100:00080000:3.0:1761315200.733303:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315200.733308:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322491200:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315200.733309:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322490816:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315200.733323:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00002000:00000024:2.0:1761315200.733327:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:3.0:1761315200.733327:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315200.733333:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322490880:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315200.733379:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322491200:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (534us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315200.733389:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 918 00000100:00100000:3.0:1761315200.733404:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dfe7af88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491200:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315201.755490:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315202.762893:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315202.762943:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315202.763004:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315202.763621:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307536768 00000100:00100000:2.0:1761315202.763657:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1537 00000100:00100000:2.0:1761315202.763661:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307536768:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315202.763707:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307536768:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 44us (765us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315202.763714:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1537 00000100:00100000:0.0:1761315202.765873:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315203.291654:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322491328:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315203.291732:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315203.291882:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491328 00000100:00100000:2.0:1761315203.291932:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2578 00000100:00100000:2.0:1761315203.291938:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322491328:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315203.291971:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315203.292026:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322491328:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (296us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315203.292033:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2578 00000100:00100000:0.0:1761315203.292150:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322491328:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315203.803889:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315204.827554:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315205.854027:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315205.854033:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315205.854037:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315205.854043:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3448 00000100:00080000:2.0:1761315205.854046:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315205.854051:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315205.854066:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000058d40a7c x1846869322491840/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315205.854079:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315205.854082:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315205.854084:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315205.854086:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3448 00000100:00080000:2.0:1761315205.854088:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315205.854090:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315205.854100:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c4df4f50 x1846869322491904/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315205.854150:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322491520:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854208:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854232:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322491776:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854254:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854268:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322492160:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854289:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854309:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315205.854313:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322491584:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854363:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854376:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322491968:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854394:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854412:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322491840:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854429:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854439:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c4df4f50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322491904:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854451:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.854475:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491520 00000100:00100000:2.0:1761315205.854507:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1538 00000100:00100000:2.0:1761315205.854512:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322491520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854548:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322491520:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (340us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.854554:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1538 00000100:00100000:2.0:1761315205.854586:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491776 00000100:00100000:2.0:1761315205.854609:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2579 00000100:00100000:2.0:1761315205.854613:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322491776:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854639:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322491776:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (384us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.854644:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2579 00010000:00010000:0.0:1761315205.854648:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315205.854663:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491584 00000100:00100000:0.0:1761315205.854688:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492160 00000100:00100000:2.0:1761315205.854699:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491968 00000100:00100000:0.0:1761315205.854722:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491840 00000100:00100000:2.0:1761315205.854728:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4107 00000100:00100000:2.0:1761315205.854732:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322491584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.854742:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491904 00000100:00100000:0.0:1761315205.854760:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1378 00000100:00100000:0.0:1761315205.854766:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322492160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854766:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322491584:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.854778:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4107 00000100:00100000:2.0:1761315205.854791:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4108 00000100:00100000:2.0:1761315205.854795:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000050474828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322491968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.854809:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322492160:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (518us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315205.854816:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1378 00000100:00100000:2.0:1761315205.854827:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000050474828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322491968:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (432us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.854835:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4108 00000100:00100000:0.0:1761315205.854845:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2015 00000100:00100000:0.0:1761315205.854851:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322491840:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854878:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322491584:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854886:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322491968:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854892:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322491520:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854898:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322491776:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854906:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322492160:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.854940:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2016 00000100:00100000:2.0:1761315205.854943:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322491904:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315205.855012:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315205.855037:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322491904:12345-0@lo:8:kworker/u8:0.0 Request processed in 93us (586us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315205.855042:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2016 00000100:00100000:2.0:1761315205.855056:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a08ec369 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491456:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315205.855074:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:0.0:1761315205.855079:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315205.855086:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322491712:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.855112:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.855125:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492096:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.855143:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322491840:12345-0@lo:8:kworker/u8:0.0 Request processed in 280us (702us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315205.855148:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315205.855155:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2015 00000100:00100000:2.0:1761315205.855169:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1761315205.855171:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492096 00000100:00100000:2.0:1761315205.855177:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491392:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315205.855205:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1379 00000100:00100000:2.0:1761315205.855205:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315205.855211:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322492096:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.855220:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491648:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.855251:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322492096:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (104us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.855252:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315205.855260:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1379 00000100:00100000:2.0:1761315205.855272:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322492032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.855295:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315205.855318:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492096:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.855322:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491392 00000100:00100000:2.0:1761315205.855366:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4109 00000100:00100000:2.0:1761315205.855372:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322491712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315205.855382:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 919 00000100:00100000:0.0:1761315205.855392:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322491392:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315205.855406:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322491712:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (292us total) trans 0 rc 0/0 00002000:00000024:0.0:1761315205.855415:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315205.855415:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4109 00000100:00100000:2.0:1761315205.855439:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491456 00000100:00100000:2.0:1761315205.855459:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322491648 00000100:00100000:0.0:1761315205.855472:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322491392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.855477:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2580 00000100:00100000:0.0:1761315205.855482:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 919 00000100:00100000:2.0:1761315205.855483:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322491456:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315205.855503:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:0.0:1761315205.855517:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315205.855530:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322491456:12345-0@lo:41:osp-pre-0-1.0 Request processed in 47us (455us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315205.855535:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2580 00000100:00100000:2.0:1761315205.855574:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491392:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315205.855588:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491648:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.855597:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322492032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315205.855608:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a08ec369 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322491456:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315205.855615:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322491712:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315205.855719:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315205.855725:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322491840:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315205.855740:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315205.855742:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315205.855745:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c4df4f50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322491904:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315206.875715:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315207.884501:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315207.886940:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315207.886984:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315207.887040:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537088 00000100:00100000:3.0:1761315207.887079:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315207.887083:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4111 00000100:00100000:2.0:1761315207.887091:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f353c47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315207.887124:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537152 00000100:00100000:3.0:1761315207.887153:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4112 00000100:00100000:2.0:1761315207.887158:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f353c47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315207.887163:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307537152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315207.887170:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4111 00000100:00100000:3.0:1761315207.887202:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307537152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315207.887210:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4112 00000100:00100000:0.0:1761315207.888424:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537024 00000100:00100000:0.0:1761315207.888469:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1539 00000100:00100000:0.0:1761315207.888474:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307537024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315207.888525:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307537024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 50us (4031us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315207.888532:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1539 00010000:00010000:2.0:1761315207.899522:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315208.411559:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8081244 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322492224:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315208.411624:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315208.411801:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492224 00000100:00100000:3.0:1761315208.411858:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2582 00000100:00100000:3.0:1761315208.411865:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322492224:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315208.411898:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315208.414160:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322492224:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2290us (2532us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315208.414172:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2582 00000100:00100000:2.0:1761315208.414255:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8081244 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322492224:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315208.923523:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315209.949809:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315210.973583:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315210.973606:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492288:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315210.973682:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315210.973730:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492288 00000100:00100000:2.0:1761315210.973772:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2583 00000100:00100000:2.0:1761315210.973778:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322492288:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315210.973807:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315210.973842:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322492288:12345-0@lo:41:osp-pre-0-1.0 Request processed in 64us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315210.973849:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2583 00000100:00100000:2.0:1761315210.973890:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492288:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315210.975443:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c6196fc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322492544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975490:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315210.975507:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322492928:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975526:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315210.975545:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315210.975550:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322492352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975574:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315210.975585:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c99b3fbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322492736:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975605:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315210.975635:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492736 00000100:00100000:3.0:1761315210.975670:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4116 00000100:00100000:3.0:1761315210.975674:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322492736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975711:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322492736:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (106us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315210.975718:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4116 00000100:00100000:3.0:1761315210.975740:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492864 00000100:00100000:3.0:1761315210.975759:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492928 00000100:00100000:3.0:1761315210.975774:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1380 00000100:00100000:3.0:1761315210.975778:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322492864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975813:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322492864:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (1515us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315210.975819:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1380 00000100:00100000:3.0:1761315210.975829:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1381 00000100:00100000:3.0:1761315210.975832:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322492928:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975856:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322492928:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (330us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315210.975860:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1381 00000100:00100000:3.0:1761315210.975887:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c99b3fbd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322492736:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975897:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322492928:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.975927:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c773eb74 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322492864:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315210.976117:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000349be38c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322492608:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315210.976145:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315210.976174:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322492672:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315210.976193:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315210.976394:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315210.976424:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315210.979991:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322492352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315210.980011:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c6196fc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322492544:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315210.981596:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322492992 00000100:00100000:0.0:1761315210.981667:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 920 00000100:00100000:0.0:1761315210.981673:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c4df4f50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322492992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315210.981702:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315210.981778:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c4df4f50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322492992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (5354us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315210.981784:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 920 00000100:00100000:3.0:1761315210.981844:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322492992:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315211.996733:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315213.000171:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315213.000214:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315213.000247:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537472 00000100:00100000:3.0:1761315213.000247:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315213.000295:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2020 00000100:00100000:0.0:1761315213.000302:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307537472:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315213.000340:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537408 00000100:00100000:3.0:1761315213.000374:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315213.000399:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537344 00010000:02020000:0.0:1761315213.000422:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315213.000442:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4117 00000100:00100000:3.0:1761315213.000449:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c773eb74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307537408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315213.000458:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4118 00000100:00100000:2.0:1761315213.000464:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315213.000483:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307537472:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 180us (320us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315213.000490:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2020 00000100:00100000:3.0:1761315213.000495:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c773eb74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307537408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 46us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315213.000500:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 33us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315213.000513:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4118 00000100:00100000:3.0:1761315213.000813:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4117 00010000:00010000:0.0:1761315213.020599:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315213.532100:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493056 00000100:00100000:3.0:1761315213.532151:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2585 00000100:00100000:3.0:1761315213.532156:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322493056:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315213.532186:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315213.532240:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322493056:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315213.532246:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2585 00000100:00100000:2.0:1761315213.532328:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322493056:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315215.069034:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315216.091715:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315216.091721:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315216.091724:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315216.091731:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3458 00000100:00080000:2.0:1761315216.091733:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315216.091739:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315216.091760:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000038772896 x1846869322493504/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315216.091779:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315216.091783:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315216.091784:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315216.091789:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3458 00000100:00080000:2.0:1761315216.091793:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315216.091797:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315216.091812:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001eebd285 x1846869322493568/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315216.091889:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322493248:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.091965:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.091996:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322493632:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092034:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.092055:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1761315216.092061:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322493184:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092102:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.092125:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322493440:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092167:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.092181:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085b3c147 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322493824:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092208:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.092246:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493376 00000100:00100000:2.0:1761315216.092276:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493248 00000100:00100000:2.0:1761315216.092296:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493632 00000100:00100000:2.0:1761315216.092313:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4119 00000100:00100000:2.0:1761315216.092318:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c99b3fbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322493376:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315216.092403:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493824 00000100:00100000:3.0:1761315216.092423:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493504 00000100:00100000:3.0:1761315216.092437:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1382 00000100:00100000:3.0:1761315216.092440:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322493824:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315216.092587:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2021 00000100:00100000:0.0:1761315216.092591:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5bc8301 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322493504:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:0.0:1761315216.092689:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315216.092770:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c99b3fbd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322493376:12345-0@lo:400:kworker/u8:1.0 Request processed in 450us (893us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.092777:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4119 00000100:00100000:2.0:1761315216.092790:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4120 00000100:00100000:2.0:1761315216.092793:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd8cc076 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322493248:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092822:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd8cc076 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322493248:12345-0@lo:400:kworker/u8:1.0 Request processed in 28us (858us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.092827:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4120 00000100:00100000:2.0:1761315216.092848:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4121 00000100:00100000:2.0:1761315216.092852:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000763e4b9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322493632:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092875:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000763e4b9f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322493632:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (842us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.092880:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4121 00000100:00100000:2.0:1761315216.092938:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322493440:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092949:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abf11666 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322493248:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.092955:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322493632:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093111:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.093132:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322493376:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093140:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bbe5509 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322493312:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093149:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322493120:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315216.093167:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315216.093172:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322493760:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093192:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.093211:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493696 00000100:00100000:2.0:1761315216.093229:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4122 00000100:00100000:2.0:1761315216.093232:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014ba56f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322493696:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093256:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014ba56f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322493696:12345-0@lo:400:kworker/u8:1.0 Request processed in 23us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.093260:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4122 00000100:00100000:2.0:1761315216.093274:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493760 00000100:00100000:2.0:1761315216.093287:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1383 00000100:00100000:2.0:1761315216.093290:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322493760:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093308:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322493760:12345-0@lo:400:kworker/u8:1.0 Request processed in 18us (116us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.093313:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1383 00000100:00100000:2.0:1761315216.093329:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322493760:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093337:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322493696:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093419:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315216.093425:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002bdc8772 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322493888:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315216.093447:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315216.093477:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493888 00000100:00100000:2.0:1761315216.093495:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 921 00000100:00100000:2.0:1761315216.093497:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9314051 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322493888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315216.093514:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315216.093544:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9314051 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322493888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.093550:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 921 00000100:00100000:2.0:1761315216.093569:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493184 00000100:00100000:2.0:1761315216.093589:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1542 00000100:00100000:2.0:1761315216.093593:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322493184:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315216.093754:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322493184:12345-0@lo:400:kworker/u8:1.0 Request processed in 160us (1657us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315216.093761:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1542 00000100:00100000:2.0:1761315216.093797:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002bdc8772 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322493888:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315216.093814:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec3fd594 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322493184:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315216.094054:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322493824:12345-0@lo:400:kworker/u8:1.0 Request processed in 1611us (1844us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315216.094068:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1382 00000100:00100000:3.0:1761315216.094088:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493568 00000100:00100000:3.0:1761315216.094113:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2022 00000100:00100000:3.0:1761315216.094118:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322493568:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:3.0:1761315216.094199:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315216.094318:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085b3c147 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322493824:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315216.094487:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322493568:12345-0@lo:8:kworker/u8:1.0 Request processed in 368us (428us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315216.094494:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2022 00000100:00100000:0.0:1761315216.094855:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5bc8301 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322493504:12345-0@lo:8:kworker/u8:1.0 Request processed in 2261us (2523us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315216.094863:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2021 00010000:00010000:0.0:1761315216.095048:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315218.120696:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315218.120880:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537728 00000100:00100000:0.0:1761315218.120930:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2023 00000100:00100000:0.0:1761315218.120938:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307537728:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315218.121033:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315218.121102:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307537728:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 163us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315218.121110:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2023 00000100:00100000:2.0:1761315218.124249:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315218.124290:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315218.124328:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315218.124918:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537536 00000100:00100000:3.0:1761315218.124950:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1543 00000100:00100000:3.0:1761315218.124955:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307537536:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315218.124996:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307537536:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 40us (754us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315218.125001:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1543 00010000:00010000:0.0:1761315218.144523:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315218.652975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322493952:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315218.653049:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315218.653247:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322493952 00000100:00100000:0.0:1761315218.653300:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2589 00000100:00100000:0.0:1761315218.653306:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f903cab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322493952:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315218.653337:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315218.653417:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f903cab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322493952:12345-0@lo:41:osp-pre-1-0.0 Request processed in 109us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315218.653424:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2589 00010000:00010000:0.0:1761315219.163517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315221.211712:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000561c506d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322494016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315221.211880:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315221.211885:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315221.211894:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315221.211898:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315221.211904:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3463 00000100:00080000:0.0:1761315221.211907:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315221.211912:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315221.211926:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322494336:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315221.211933:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006e65d9e2 x1846869322494464/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315221.211949:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315221.211953:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315221.211954:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315221.211958:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3463 00000100:00080000:0.0:1761315221.211960:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315221.211967:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315221.211976:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494016 00000100:00100000:0.0:1761315221.211982:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006536d946 x1846869322494528/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315221.212025:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 922 00000100:00100000:2.0:1761315221.212034:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322494656:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212037:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322494016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315221.212070:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315221.212080:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315221.212106:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315221.212123:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322494016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 88us (244us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315221.212133:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 922 00000100:00100000:2.0:1761315221.212140:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494208 00000100:00100000:0.0:1761315221.212142:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494656 00000100:00100000:2.0:1761315221.212181:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494592 00000100:00100000:3.0:1761315221.212187:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315221.212193:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4125 00000100:00100000:0.0:1761315221.212200:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322494208:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212219:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494336 00000100:00100000:3.0:1761315221.212241:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000561c506d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322494016:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315221.212249:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4126 00000100:00100000:0.0:1761315221.212253:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322494208:12345-0@lo:400:kworker/u8:1.0 Request processed in 52us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315221.212260:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024ebb76f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322494656:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315221.212265:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4125 00000100:00100000:3.0:1761315221.212268:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322494720:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315221.212289:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4127 00000100:00100000:0.0:1761315221.212294:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322494592:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212316:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315221.212331:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024ebb76f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322494656:12345-0@lo:400:kworker/u8:1.0 Request processed in 73us (252us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315221.212362:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4126 00000100:00100000:0.0:1761315221.212366:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322494592:12345-0@lo:400:kworker/u8:1.0 Request processed in 70us (251us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315221.212375:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4127 00000100:00100000:3.0:1761315221.212382:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315221.212386:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4128 00000100:00100000:2.0:1761315221.212394:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322494336:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212394:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081d868c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322494144:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315221.212411:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322494464:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212429:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315221.212451:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322494400:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212463:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322494336:12345-0@lo:400:kworker/u8:1.0 Request processed in 69us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315221.212474:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4128 00000100:00100000:3.0:1761315221.212492:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315221.212497:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315221.212504:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494144 00000100:00100000:0.0:1761315221.212537:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006536d946 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322494528:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212541:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1544 00000100:00100000:2.0:1761315221.212549:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322494144:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212553:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322494336:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212608:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322494144:12345-0@lo:400:kworker/u8:1.0 Request processed in 55us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315221.212609:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315221.212619:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1544 00000100:00100000:3.0:1761315221.212626:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081d868c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322494144:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212642:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322494400:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212677:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322494656:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212707:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494528 00000100:00100000:0.0:1761315221.212713:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494464 00000100:00100000:2.0:1761315221.212727:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322494592:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212742:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322494272:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212756:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8081244 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322494208:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315221.212761:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2024 00000100:00100000:3.0:1761315221.212769:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322494464:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315221.212777:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2025 00000100:00100000:2.0:1761315221.212778:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322494080:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315221.212786:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322494528:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315221.212850:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322494720:0@lo:400:kworker/u8:1.0 00010000:02020000:3.0:1761315221.212945:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315221.212986:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322494464:12345-0@lo:8:kworker/u8:1.0 Request processed in 219us (491us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315221.212993:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2024 00010000:02020000:0.0:1761315221.213041:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315221.213078:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322494528:12345-0@lo:8:kworker/u8:1.0 Request processed in 293us (473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315221.213084:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2025 00010000:00010000:0.0:1761315222.235614:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315223.240942:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315223.240987:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315223.241026:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315223.241122:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537856 00000100:00100000:0.0:1761315223.241192:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4129 00000100:00100000:0.0:1761315223.241199:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d48bf7ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315223.241274:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d48bf7ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307537856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 74us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315223.241282:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4129 00000100:00100000:3.0:1761315223.242581:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537792 00000100:00100000:3.0:1761315223.242632:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1545 00000100:00100000:3.0:1761315223.242639:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307537792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315223.242704:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307537792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (1677us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315223.242711:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1545 00000100:00100000:3.0:1761315223.242769:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315223.242856:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307537920 00000100:00100000:3.0:1761315223.242880:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4130 00000100:00100000:3.0:1761315223.242884:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307537920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315223.242913:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307537920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 28us (145us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315223.242919:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4130 00000100:00100000:2.0:1761315223.773249:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322494784:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315223.773316:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315223.773644:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322494784:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315224.283649:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315225.307502:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315226.331824:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322494848:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.331934:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315226.332454:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494912 00000100:00100000:3.0:1761315226.332492:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4131 00000100:00100000:3.0:1761315226.332497:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322494912:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333483:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322494912:12345-0@lo:400:kworker/u8:1.0 Request processed in 983us (1204us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315226.333494:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4131 00000100:00100000:3.0:1761315226.333527:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322494976:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333570:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315226.333594:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315226.333599:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322495040:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333621:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315226.333646:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494976 00000100:00100000:3.0:1761315226.333673:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2594 00000100:00100000:3.0:1761315226.333678:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322494976:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333711:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322494976:12345-0@lo:400:kworker/u8:1.0 Request processed in 33us (141us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315226.333717:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2594 00000100:00100000:3.0:1761315226.333734:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495040 00000100:00100000:3.0:1761315226.333752:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4132 00000100:00100000:3.0:1761315226.333756:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322495040:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333781:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322495040:12345-0@lo:400:kworker/u8:1.0 Request processed in 25us (161us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315226.333787:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4132 00000100:00100000:3.0:1761315226.333822:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322495040:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333846:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322494976:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333872:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322495104:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333893:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315226.333916:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495104 00000100:00100000:3.0:1761315226.333932:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2595 00000100:00100000:3.0:1761315226.333936:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322495104:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315226.333961:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322495104:12345-0@lo:400:kworker/u8:1.0 Request processed in 24us (68us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315226.333966:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2595 00000100:00100000:3.0:1761315226.333984:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322495104:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315226.334123:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322494848 00000100:00100000:2.0:1761315226.334146:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1546 00000100:00100000:2.0:1761315226.334149:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322494848:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315226.334178:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322494848:12345-0@lo:400:kworker/u8:1.0 Request processed in 28us (2246us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315226.334183:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1546 00000100:00100000:2.0:1761315226.334211:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322494848:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315226.334219:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315226.334223:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ab44e19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322495168:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315226.334243:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315226.334264:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495168 00000100:00100000:2.0:1761315226.334286:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 923 00000100:00100000:2.0:1761315226.334290:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002692a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322495168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315226.334309:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315226.334339:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002692a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322495168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (96us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315226.334360:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 923 00000100:00100000:2.0:1761315226.334389:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ab44e19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322495168:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315226.334501:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315226.334506:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315226.334509:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315226.334514:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3468 00000100:00080000:0.0:1761315226.334516:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315226.334520:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315226.334549:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002e53cb60 x1846869322495232/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315226.334562:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315226.334565:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315226.334567:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315226.334569:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3468 00000100:00080000:0.0:1761315226.334571:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315226.334574:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315226.334584:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000001339376 x1846869322495296/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315226.334661:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322495424:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315226.334689:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315226.334703:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315226.334707:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322495360:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315226.334725:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315226.334746:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495424 00000100:00100000:0.0:1761315226.334766:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495360 00000100:00100000:0.0:1761315226.334781:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4133 00000100:00100000:0.0:1761315226.334785:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322495424:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315226.334814:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322495424:12345-0@lo:400:kworker/u8:1.0 Request processed in 28us (124us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315226.334819:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4133 00000100:00100000:0.0:1761315226.334829:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4134 00000100:00100000:0.0:1761315226.335497:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322495360:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315226.335525:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322495360:12345-0@lo:400:kworker/u8:1.0 Request processed in 29us (799us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315226.335575:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322495232:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315226.335597:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315226.335610:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322495296:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315226.335626:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315226.335649:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495488 00000100:00100000:2.0:1761315226.335667:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495232 00000100:00100000:2.0:1761315226.335676:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495296 00000100:00100000:2.0:1761315226.335687:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1385 00000100:00100000:2.0:1761315226.335689:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322495488:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315226.335714:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322495488:12345-0@lo:400:kworker/u8:1.0 Request processed in 24us (761us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315226.335718:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1385 00000100:00100000:2.0:1761315226.335728:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2027 00000100:00100000:2.0:1761315226.335731:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ab44e19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322495232:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315226.335824:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315226.335861:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ab44e19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322495232:12345-0@lo:8:kworker/u8:1.0 Request processed in 128us (263us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315226.335867:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2027 00000100:00100000:2.0:1761315226.335876:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2028 00000100:00100000:2.0:1761315226.335878:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d7c0a8d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322495296:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315226.335948:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315226.335967:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d7c0a8d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322495296:12345-0@lo:8:kworker/u8:1.0 Request processed in 87us (340us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315226.335972:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2028 00000100:00080000:2.0:1761315226.336007:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315226.336010:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315226.336013:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322495232:0@lo:8:kworker/u8:1.0 00000100:00080000:2.0:1761315226.336021:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315226.336023:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315226.336025:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322495296:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315226.336560:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4134 00000100:00100000:2.0:1761315226.336698:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495552 00000100:00100000:2.0:1761315226.336734:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2596 00000100:00100000:2.0:1761315226.336737:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322495552:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315226.336761:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315226.337695:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322495552:12345-0@lo:41:osp-pre-0-1.0 Request processed in 953us (1370us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315226.337703:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2596 00010000:00010000:0.0:1761315227.355536:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315228.360320:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315228.360445:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315228.360472:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538048 00000100:00100000:0.0:1761315228.360670:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538240 00000100:00100000:0.0:1761315228.360709:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2029 00000100:00100000:2.0:1761315228.360713:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1547 00000100:00100000:0.0:1761315228.360718:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307538240:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315228.360723:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315228.360812:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 90us (506us total) trans 0 rc 0/0 00010000:02020000:0.0:1761315228.360817:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315228.360822:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1547 00000100:00100000:0.0:1761315228.360852:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307538240:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 134us (407us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315228.360860:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2029 00000100:00100000:3.0:1761315228.361535:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315228.361627:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538176 00000100:00100000:0.0:1761315228.361671:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4135 00000100:00100000:0.0:1761315228.361676:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315228.361728:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 52us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315228.361735:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4135 00000100:00100000:2.0:1761315228.362157:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315228.362321:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538112 00000100:00100000:3.0:1761315228.362398:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4136 00000100:00100000:3.0:1761315228.362403:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b1a4179 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315228.362479:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b1a4179 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 74us (322us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315228.362486:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4136 00010000:00010000:0.0:1761315228.379589:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315228.892233:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322495616:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315228.892336:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315228.892553:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495616 00000100:00100000:2.0:1761315228.892606:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2597 00000100:00100000:2.0:1761315228.892611:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322495616:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315228.892658:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315228.892709:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322495616:12345-0@lo:41:osp-pre-1-0.0 Request processed in 97us (374us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315228.892718:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2597 00000100:00100000:3.0:1761315228.892750:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322495616:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315230.427498:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315231.451680:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495680 00000100:00100000:3.0:1761315231.451726:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 924 00000100:00100000:3.0:1761315231.451733:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322495680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315231.451760:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315231.451911:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322495680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 177us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315231.451918:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 924 00000100:00100000:3.0:1761315231.452876:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495744 00000100:00080000:2.0:1761315231.453382:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315231.453388:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315231.453391:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315231.453396:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3473 00000100:00080000:2.0:1761315231.453399:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315231.453403:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315231.453420:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000be6b3021 x1846869322496064/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315231.453434:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315231.453437:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315231.453438:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315231.453441:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3473 00000100:00080000:2.0:1761315231.453443:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315231.453445:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315231.453455:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b5c22524 x1846869322496128/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315231.453489:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be6b3021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322496064:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315231.453543:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315231.453565:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5c22524 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322496128:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315231.453585:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315231.453780:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2598 00000100:00100000:0.0:1761315231.453786:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322495744:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315231.453813:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315231.453846:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322495744:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (1847us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315231.453855:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2598 00000100:00100000:0.0:1761315231.453897:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496064 00000100:00100000:0.0:1761315231.453918:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496128 00000100:00100000:0.0:1761315231.453932:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2030 00000100:00100000:0.0:1761315231.453936:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322496064:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315231.454023:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315231.454041:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2031 00000100:00100000:2.0:1761315231.454047:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322496128:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315231.454071:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322496064:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (527us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315231.454078:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2030 00000100:00100000:0.0:1761315231.454097:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088299d09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322496000:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315231.454123:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:2.0:1761315231.454137:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315231.454142:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315231.454151:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000015b27f3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322495808:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454177:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322496128:12345-0@lo:8:kworker/u8:0.0 Request processed in 128us (590us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315231.454185:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2031 00000100:00100000:0.0:1761315231.454189:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315231.454209:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006be81110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322496192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454218:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322495808 00000100:00100000:0.0:1761315231.454238:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315231.454252:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1761315231.454258:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322495808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315231.454272:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496000 00000100:00100000:0.0:1761315231.454306:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2599 00000100:00100000:2.0:1761315231.454309:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322495808:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (120us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315231.454314:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322496000:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454322:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1548 00000100:00100000:2.0:1761315231.454385:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496320 00000100:00100000:2.0:1761315231.454416:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1386 00000100:00100000:2.0:1761315231.454421:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322496320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454469:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322496320:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315231.454477:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1386 00000100:00080000:2.0:1761315231.454529:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315231.454533:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315231.454538:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be6b3021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322496064:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315231.454551:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315231.454554:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315231.454557:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5c22524 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322496128:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454590:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000015b27f3c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322495808:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454727:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006be81110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322496192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454786:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4140 00000100:00100000:2.0:1761315231.454791:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322496256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.454824:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322496256:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315231.454870:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322496000:12345-0@lo:400:kworker/u8:0.0 Request processed in 557us (745us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315231.454877:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2599 00000100:00100000:3.0:1761315231.455038:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088299d09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322496000:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315231.455319:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4140 00000100:00100000:0.0:1761315231.455365:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000690c03d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322495872:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315231.455382:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000561c506d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322496256:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315231.455391:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093ab9157 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322495936:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315233.480918:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315233.480969:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315233.481005:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315233.481047:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315233.481089:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538496 00000100:00100000:2.0:1761315233.481129:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2032 00000100:00100000:2.0:1761315233.481136:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307538496:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:2.0:1761315233.481234:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315233.481299:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307538496:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 161us (250us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315233.481307:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2032 00000100:00100000:0.0:1761315233.482098:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538368 00000100:00100000:0.0:1761315233.482135:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538432 00000100:00100000:0.0:1761315233.482413:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4141 00000100:00100000:0.0:1761315233.482419:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538368:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315233.482459:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538368:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 40us (1549us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315233.482466:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4141 00000100:00100000:0.0:1761315233.482477:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4142 00000100:00100000:0.0:1761315233.482499:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538432:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315233.484035:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538432:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 1530us (3025us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315233.484061:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4142 00010000:00010000:0.0:1761315233.500588:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315234.011738:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322496384:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315234.011814:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315234.011867:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496384 00000100:00100000:2.0:1761315234.011924:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2600 00000100:00100000:2.0:1761315234.011929:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322496384:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315234.011961:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315234.012015:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322496384:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (203us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315234.012023:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2600 00000100:00100000:3.0:1761315234.012242:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322496384:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315234.524834:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315236.571558:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093ab9157 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322496448:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315236.571625:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315236.571687:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496448 00000100:00100000:2.0:1761315236.571741:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 925 00000100:00100000:2.0:1761315236.571748:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322496448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315236.571779:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315236.571823:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322496448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (198us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315236.571832:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 925 00000100:00100000:2.0:1761315236.571881:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093ab9157 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322496448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315236.571997:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322496576:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.572034:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315236.572034:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315236.572041:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315236.572044:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315236.572048:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3479 00000100:00080000:3.0:1761315236.572051:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315236.572055:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315236.572070:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315236.572073:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a8081244 x1846869322496832/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315236.572079:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322496768:0@lo:400:kworker/u8:1.0 00000100:00080000:3.0:1761315236.572090:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315236.572094:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315236.572095:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315236.572099:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3479 00000100:00080000:3.0:1761315236.572101:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315236.572103:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315236.572108:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315236.572116:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002731a333 x1846869322496896/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315236.572164:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081d868c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322496960:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.572171:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496576 00000100:00100000:3.0:1761315236.572197:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315236.572200:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1550 00000100:00100000:0.0:1761315236.572206:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322496576:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.572223:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322496704:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.572249:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322496576:12345-0@lo:400:kworker/u8:1.0 Request processed in 42us (215us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315236.572254:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315236.572265:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1550 00000100:00100000:3.0:1761315236.572273:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322497088:0@lo:400:kworker/u8:1.0 00010000:00010000:0.0:1761315236.572295:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315236.572299:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315236.572314:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322496576:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.572317:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315236.572325:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322496640:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.572370:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315236.572372:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496960 00000100:00100000:3.0:1761315236.572387:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322497024:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.572400:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496704 00000100:00100000:3.0:1761315236.572411:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315236.572504:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497088 00000100:00100000:3.0:1761315236.572529:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1387 00000100:00100000:3.0:1761315236.572532:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322497088:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.572562:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322497088:12345-0@lo:400:kworker/u8:1.0 Request processed in 29us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315236.572568:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1387 00000100:00100000:2.0:1761315236.572704:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8081244 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322496832:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315236.572729:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315236.572742:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322496896:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315236.572757:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315236.572779:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496832 00000100:00100000:2.0:1761315236.572793:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496896 00000100:00100000:2.0:1761315236.572805:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2033 00000100:00100000:2.0:1761315236.572809:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322496832:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315236.572898:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315236.572926:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322496832:12345-0@lo:8:kworker/u8:1.0 Request processed in 116us (197us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315236.572932:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2033 00000100:00100000:2.0:1761315236.572941:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2034 00000100:00100000:2.0:1761315236.572944:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322496896:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315236.573021:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315236.573038:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322496896:12345-0@lo:8:kworker/u8:1.0 Request processed in 93us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315236.573047:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2034 00000100:00080000:2.0:1761315236.573072:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315236.573074:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315236.573077:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8081244 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322496832:0@lo:8:kworker/u8:1.0 00000100:00080000:2.0:1761315236.573084:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315236.573086:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315236.573088:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322496896:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315236.573728:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322496640 00000100:00100000:3.0:1761315236.573730:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497024 00000100:00100000:2.0:1761315236.573753:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4144 00000100:00100000:2.0:1761315236.573764:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322496704:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.573774:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4145 00000100:00100000:0.0:1761315236.573779:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322496640:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315236.573820:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322496704:12345-0@lo:400:kworker/u8:1.0 Request processed in 57us (1568us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315236.573829:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322496640:12345-0@lo:400:kworker/u8:1.0 Request processed in 48us (1457us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315236.573835:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4144 00000100:00100000:0.0:1761315236.573841:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4145 00000100:00100000:2.0:1761315236.573893:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322496640:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315236.573917:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322496704:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.573926:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081d868c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322496960:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315236.573944:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322497088:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315236.573958:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322496768:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.573962:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4146 00000100:00100000:3.0:1761315236.573972:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322497024:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315236.574376:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322497024:12345-0@lo:400:kworker/u8:1.0 Request processed in 406us (1965us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315236.574383:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4146 00000100:00100000:2.0:1761315236.574422:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d466d0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322497024:0@lo:400:kworker/u8:1.0 00010000:00010000:0.0:1761315237.595996:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315238.603587:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538688 00000100:00100000:3.0:1761315238.603639:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538624 00000100:00100000:3.0:1761315238.603662:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4147 00000100:00100000:3.0:1761315238.603667:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000690c03d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538688:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315238.603741:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000690c03d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538688:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 66us (252us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315238.603749:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4147 00000100:00100000:3.0:1761315238.603769:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4148 00000100:00100000:3.0:1761315238.603774:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088299d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315238.603806:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088299d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 30us (287us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315238.603814:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4148 00000100:00100000:0.0:1761315238.603974:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538560 00000100:00100000:0.0:1761315238.604013:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1551 00000100:00100000:0.0:1761315238.604018:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315238.604060:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (659us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315238.604070:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1551 00010000:00010000:2.0:1761315238.619817:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315239.134614:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497152 00000100:00100000:0.0:1761315239.134675:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2603 00000100:00100000:0.0:1761315239.134681:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015b27f3c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322497152:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315239.134718:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315239.135035:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015b27f3c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322497152:12345-0@lo:41:osp-pre-1-0.0 Request processed in 353us (1616us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315239.135043:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2603 00000100:00100000:2.0:1761315239.135111:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322497152:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315239.644037:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315240.667520:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315241.691602:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315241.691621:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322497216:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315241.691690:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315241.691842:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2604 00000100:00100000:3.0:1761315241.691848:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322497216:12345-0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315241.691852:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000004:00000024:3.0:1761315241.691878:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315241.691916:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322497344:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315241.691923:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322497216:12345-0@lo:41:osp-pre-0-1.0 Request processed in 72us (232us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315241.691931:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2604 00000100:00080000:0.0:1761315241.691946:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315241.691950:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315241.691953:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315241.691959:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3484 00000100:00080000:0.0:1761315241.691963:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315241.691963:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315241.691969:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315241.691989:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a0758582 x1846869322497600/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315241.691989:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315241.691999:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322497536:0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315241.692009:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315241.692013:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315241.692015:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315241.692020:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3484 00000100:00080000:0.0:1761315241.692027:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315241.692027:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315241.692033:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315241.692048:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f9b1c2df x1846869322497664/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315241.692058:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322497472:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315241.692094:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315241.692114:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497344 00000100:00100000:2.0:1761315241.692119:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064ec7a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322497856:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315241.692150:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 926 00000100:00100000:0.0:1761315241.692155:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322497344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315241.692176:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315241.692324:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f03c3dae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322497408:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315241.692676:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322497344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 519us (717us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315241.692680:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315241.692689:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 926 00000100:00100000:0.0:1761315241.692718:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322497600:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315241.692727:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497472 00000100:00100000:0.0:1761315241.692753:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315241.692759:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497536 00000100:00100000:2.0:1761315241.692764:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497728 00000100:00100000:0.0:1761315241.692782:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9b1c2df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322497664:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315241.692792:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2605 00000100:00100000:2.0:1761315241.692796:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4149 00000100:00100000:3.0:1761315241.692799:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322497536:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315241.692805:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d466d0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322497472:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315241.692812:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315241.692850:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322497536:12345-0@lo:400:kworker/u8:1.0 Request processed in 53us (824us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315241.692856:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2605 00000100:00100000:0.0:1761315241.693168:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322497536:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315241.693174:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d466d0d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322497472:12345-0@lo:400:kworker/u8:1.0 Request processed in 368us (1079us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315241.693181:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315241.693184:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4149 00000100:00100000:3.0:1761315241.693204:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322497792:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315241.693211:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497408 00000100:00100000:2.0:1761315241.693214:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4150 00000100:00100000:2.0:1761315241.693221:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5c22524 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322497728:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315241.693231:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315241.693242:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497792 00000100:00100000:3.0:1761315241.693257:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322497216:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315241.693263:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4151 00000100:00100000:0.0:1761315241.693269:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9137a0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322497408:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315241.693277:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5c22524 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322497728:12345-0@lo:400:kworker/u8:1.0 Request processed in 55us (593us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315241.693284:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4150 00000100:00100000:2.0:1761315241.693302:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4152 00000100:00100000:2.0:1761315241.693307:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322497792:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315241.693312:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2037 00000100:00100000:3.0:1761315241.693317:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba0fa64f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322497664:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:3.0:1761315241.693405:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315241.693673:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba0fa64f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322497664:12345-0@lo:8:kworker/u8:1.0 Request processed in 354us (862us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315241.693679:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2037 00000100:00100000:0.0:1761315241.693700:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9137a0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322497408:12345-0@lo:400:kworker/u8:1.0 Request processed in 430us (518us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315241.693709:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4151 00000100:00100000:3.0:1761315241.693739:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064ec7a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322497856:0@lo:400:kworker/u8:1.0 00000100:00080000:0.0:1761315241.693747:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315241.693751:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315241.693753:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322497472:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315241.693757:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322497600:0@lo:8:kworker/u8:1.0 00000100:00080000:0.0:1761315241.693772:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315241.693775:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315241.693779:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9b1c2df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322497664:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315241.693802:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322497728:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315241.694069:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322497792:12345-0@lo:400:kworker/u8:1.0 Request processed in 760us (837us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315241.694083:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4152 00010000:00010000:2.0:1761315242.716532:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315243.722816:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315243.722876:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315243.722887:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538880 00000100:00100000:3.0:1761315243.722924:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315243.722948:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538944 00000100:00100000:2.0:1761315243.722972:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4153 00000100:00100000:2.0:1761315243.722979:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538880:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315243.722994:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4154 00000100:00100000:3.0:1761315243.723007:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315243.723083:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307538944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (160us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315243.723091:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4154 00000100:00100000:2.0:1761315243.723161:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307538880:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 176us (348us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315243.723171:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4153 00000100:00100000:0.0:1761315243.723813:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307538816 00000100:00100000:0.0:1761315243.723857:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1553 00000100:00100000:0.0:1761315243.723862:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538816:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315243.723898:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307538816:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (1024us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315243.723906:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1553 00000100:00100000:3.0:1761315243.724945:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315243.725031:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539008 00000100:00100000:2.0:1761315243.725068:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2038 00000100:00100000:2.0:1761315243.725072:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307539008:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315243.725165:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315243.725212:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307539008:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 138us (269us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315243.725219:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2038 00010000:00010000:0.0:1761315243.740622:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315244.252173:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9b1c2df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322497920:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315244.252250:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315244.252304:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497920 00000100:00100000:2.0:1761315244.252418:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2606 00000100:00100000:2.0:1761315244.252425:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322497920:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315244.252462:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315244.254125:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322497920:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1693us (1871us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315244.254138:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2606 00000100:00100000:3.0:1761315244.254230:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9b1c2df pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322497920:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315244.763581:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315246.811553:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f2cd44f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322498048:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315246.811565:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315246.811572:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315246.811576:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315246.811581:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3489 00000100:00080000:0.0:1761315246.811585:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315246.811592:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315246.811617:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000292a2071 x1846869322498368/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315246.811635:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315246.811639:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315246.811640:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315246.811644:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3489 00000100:00080000:0.0:1761315246.811647:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315246.811650:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315246.811663:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070b25679 x1846869322498432/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315246.811741:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322498368:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315246.811777:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315246.811805:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070b25679 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322498432:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315246.811827:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315246.811864:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315246.811886:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498368 00000100:00100000:0.0:1761315246.811918:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498432 00000100:00100000:0.0:1761315246.811936:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2039 00000100:00100000:0.0:1761315246.811942:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322498368:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315246.812037:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315246.812077:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322498368:12345-0@lo:8:kworker/u8:0.0 Request processed in 134us (300us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315246.812083:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2039 00000100:00100000:0.0:1761315246.812102:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2040 00000100:00100000:0.0:1761315246.812106:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322498432:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315246.812187:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315246.812210:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322498432:12345-0@lo:8:kworker/u8:0.0 Request processed in 104us (382us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315246.812216:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2040 00000100:00080000:0.0:1761315246.812253:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315246.812258:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315246.812263:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322498368:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315246.812275:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315246.812277:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315246.812280:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070b25679 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322498432:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315246.812286:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322498112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315246.812318:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322498176:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.812325:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315246.812379:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbe04555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322498496:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.812449:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315246.812518:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315246.812552:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a244fd1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322498240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.812584:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498048 00000100:00100000:3.0:1761315246.812613:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2607 00000100:00100000:3.0:1761315246.812617:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322498048:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315246.812706:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315246.812766:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322498048:12345-0@lo:41:osp-pre-0-1.0 Request processed in 148us (1135us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.812772:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2607 00000100:00100000:2.0:1761315246.813742:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322498752:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315246.813743:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322497984 00000100:00100000:2.0:1761315246.813780:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315246.813780:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1554 00000100:00100000:3.0:1761315246.813788:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322497984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.813831:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322497984:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (499us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.813838:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1554 00000100:00100000:3.0:1761315246.813865:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498304 00000100:00100000:3.0:1761315246.813887:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498176 00000100:00100000:3.0:1761315246.813902:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2608 00000100:00100000:3.0:1761315246.813906:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be6b3021 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322498304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.813935:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be6b3021 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322498304:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (530us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.813940:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2608 00000100:00100000:3.0:1761315246.813950:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2609 00000100:00100000:3.0:1761315246.813952:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093ab9157 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322498176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.813977:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093ab9157 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322498176:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (361us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.813982:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2609 00000100:00100000:3.0:1761315246.814012:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315246.814111:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498112 00000100:00100000:3.0:1761315246.814113:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f98a942 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322498624:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.814115:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498496 00000100:00100000:3.0:1761315246.814145:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315246.814146:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498240 00000100:00100000:2.0:1761315246.814158:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4155 00000100:00100000:2.0:1761315246.814165:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8081244 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322498112:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.814181:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f2cd44f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322498048:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315246.814187:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4156 00000100:00100000:0.0:1761315246.814197:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec3fd594 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322498496:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.814213:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8081244 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322498112:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (1890us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.814218:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498624 00000100:00100000:2.0:1761315246.814219:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4155 00000100:00100000:0.0:1761315246.814250:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec3fd594 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322498496:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (1800us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315246.814251:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498752 00000100:00100000:3.0:1761315246.814255:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1390 00000100:00100000:0.0:1761315246.814261:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4156 00000100:00100000:3.0:1761315246.814265:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322498624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.814282:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 927 00000100:00100000:2.0:1761315246.814286:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bbe5509 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322498752:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315246.814306:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322498624:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (161us total) trans 0 rc 0/0 00002000:00000024:2.0:1761315246.814309:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315246.814317:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1390 00000100:00100000:0.0:1761315246.814321:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315246.814370:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498560 00000100:00100000:2.0:1761315246.814376:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bbe5509 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322498752:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (600us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315246.814383:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 927 00000100:00100000:0.0:1761315246.814404:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322498176:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.814415:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4158 00000100:00100000:3.0:1761315246.814420:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070b25679 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322498560:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.814445:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322498112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315246.814464:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322497984:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315246.814465:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbe04555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322498496:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315246.814467:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070b25679 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322498560:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315246.814479:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4158 00000100:00100000:2.0:1761315246.814492:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002731a333 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322498752:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315246.814493:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000509695a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322498304:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315247.836455:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315248.840475:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315248.840520:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315248.840555:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315248.840591:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539136 00000100:00100000:3.0:1761315248.840637:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539200 00000100:00100000:3.0:1761315248.840657:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4159 00000100:00100000:3.0:1761315248.840663:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f2cd44f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307539136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315248.840732:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f2cd44f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307539136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (212us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315248.840739:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4159 00000100:00100000:3.0:1761315248.840758:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4160 00000100:00100000:3.0:1761315248.840761:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307539200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315248.840786:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307539200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (231us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315248.840792:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4160 00000100:00100000:2.0:1761315248.841485:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539072 00000100:00100000:2.0:1761315248.841516:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1555 00000100:00100000:2.0:1761315248.841521:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307539072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315248.843452:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307539072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1924us (2977us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315248.843467:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1555 00000100:00100000:3.0:1761315248.843532:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315248.843595:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539264 00000100:00100000:0.0:1761315248.843642:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2041 00000100:00100000:0.0:1761315248.843647:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307539264:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315248.843738:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315248.843780:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307539264:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 132us (251us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315248.843787:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2041 00010000:00010000:2.0:1761315248.859607:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315249.371671:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498816 00000100:00100000:3.0:1761315249.371721:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2610 00000100:00100000:3.0:1761315249.371727:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322498816:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315249.371759:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315249.371809:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322498816:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (186us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315249.371817:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2610 00010000:00010000:0.0:1761315249.883544:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315250.907488:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315251.931654:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:2.0:1761315251.931654:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315251.931661:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315251.931664:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315251.931673:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3494 00000100:00100000:2.0:1761315251.931673:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322499008:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315251.931679:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315251.931685:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315251.931707:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c7ce5eac x1846869322499264/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315251.931724:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315251.931727:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315251.931729:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315251.931734:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3494 00000100:00100000:2.0:1761315251.931735:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315251.931737:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315251.931742:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315251.931755:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000078729988 x1846869322499328/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315251.931777:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000975475fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322499392:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.931793:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322498880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315251.931807:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315251.931818:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c7ce5eac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322499264:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315251.931827:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1761315251.931834:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322498944:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.931850:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315251.931863:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315251.931894:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315251.931908:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a06c7ae2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322499200:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.931927:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315251.931938:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbe04555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322499584:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.931952:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007349fc1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322499136:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.931958:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315251.931984:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315251.931989:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322498944 00000100:00100000:3.0:1761315251.932004:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3c2a916 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322499520:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932029:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1556 00000100:00100000:2.0:1761315251.932036:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322498944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932037:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315251.932061:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315251.932067:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322499072:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932090:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315251.932102:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000509695a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322499456:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932120:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315251.932142:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499264 00000100:00100000:3.0:1761315251.932161:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499584 00000100:00100000:3.0:1761315251.932177:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499520 00000100:00100000:3.0:1761315251.932190:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1391 00000100:00100000:3.0:1761315251.932194:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322499584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932228:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322499584:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315251.932235:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1391 00000100:00100000:3.0:1761315251.932248:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1392 00000100:00100000:3.0:1761315251.932251:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322499520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932277:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322499520:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (252us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315251.932282:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1392 00000100:00100000:3.0:1761315251.932310:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2042 00000100:00100000:3.0:1761315251.932313:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095e7dbc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322499264:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315251.932442:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315251.932459:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322498944:12345-0@lo:400:kworker/u8:2.0 Request processed in 422us (564us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315251.932467:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1556 00000100:00100000:3.0:1761315251.932479:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095e7dbc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322499264:12345-0@lo:8:kworker/u8:2.0 Request processed in 163us (618us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315251.932487:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2042 00000100:00100000:2.0:1761315251.932528:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499008 00000100:00100000:2.0:1761315251.932555:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499392 00000100:00100000:2.0:1761315251.932573:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499136 00000100:00100000:2.0:1761315251.932589:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499456 00000100:00100000:2.0:1761315251.932606:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4161 00000100:00100000:2.0:1761315251.932610:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abf11666 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322499008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932647:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abf11666 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322499008:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (914us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315251.932659:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4161 00000100:00100000:2.0:1761315251.932676:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4162 00000100:00100000:2.0:1761315251.932681:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002731a333 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322499392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932700:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3c2a916 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322499520:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315251.932720:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315251.932726:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315251.932727:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002731a333 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322499392:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (920us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315251.932733:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c7ce5eac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322499264:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932739:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4162 00000100:00100000:0.0:1761315251.932743:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322499328:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932747:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322498944:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932756:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4163 00000100:00100000:2.0:1761315251.932764:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322499136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932769:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbe04555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322499584:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932787:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000975475fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322499392:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315251.932788:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315251.932805:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322499008:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932808:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322499136:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (823us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315251.932818:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4163 00000100:00100000:2.0:1761315251.932843:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4164 00000100:00100000:2.0:1761315251.932849:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322499456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315251.932868:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499328 00000100:00100000:3.0:1761315251.932872:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322498880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315251.932885:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322499456:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (763us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315251.932893:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007349fc1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322499136:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932896:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4164 00000100:00100000:0.0:1761315251.932903:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2043 00000100:00100000:0.0:1761315251.932911:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f98a942 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322499328:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315251.932922:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000509695a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322499456:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932942:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2612 00000100:00100000:2.0:1761315251.932948:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322499072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315251.932988:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322499072:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (896us total) trans 0 rc 0/0 00010000:02020000:0.0:1761315251.932998:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315251.932999:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2612 00000100:00100000:3.0:1761315251.933007:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322499072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315251.933061:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f98a942 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322499328:12345-0@lo:8:kworker/u8:2.0 Request processed in 150us (275us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315251.933074:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2043 00000100:00080000:0.0:1761315251.933105:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315251.933109:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315251.933113:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322499328:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315251.933203:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499648 00000100:00100000:3.0:1761315251.933235:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2613 00000100:00100000:3.0:1761315251.933239:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322499648:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315251.933258:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315251.933303:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322499648:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315251.933309:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2613 00010000:00010000:0.0:1761315252.955648:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315253.962762:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315253.963331:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539328 00000100:00100000:2.0:1761315253.963456:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1557 00000100:00100000:2.0:1761315253.963463:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307539328:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315253.963539:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307539328:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 76us (785us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315253.963547:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1557 00000100:00100000:3.0:1761315253.964608:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315253.966028:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315253.966279:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539520 00000100:00100000:0.0:1761315253.966320:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2044 00000100:00100000:0.0:1761315253.966327:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307539520:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315253.966439:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315253.966502:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307539520:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 174us (1896us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315253.966509:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2044 00000100:00100000:3.0:1761315253.967035:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315253.969564:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539456 00000100:00100000:2.0:1761315253.972289:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4165 00000100:00100000:2.0:1761315253.972298:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307539456:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315253.972412:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307539456:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 114us (6389us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315253.972421:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4165 00010000:00010000:0.0:1761315253.980823:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315254.491812:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499712 00000100:00100000:0.0:1761315254.491866:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2614 00000100:00100000:0.0:1761315254.491872:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322499712:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315254.491904:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315254.491959:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322499712:12345-0@lo:41:osp-pre-1-0.0 Request processed in 86us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315254.491965:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2614 00000100:00100000:2.0:1761315254.494461:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322499712:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315255.004074:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315256.027798:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315257.051526:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315257.051617:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322499776:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315257.051629:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315257.051635:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315257.051638:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315257.051643:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3499 00000100:00080000:2.0:1761315257.051646:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315257.051651:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315257.051670:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000058d40a7c x1846869322500160/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315257.051684:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315257.051687:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315257.051689:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315257.051692:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3499 00000100:00080000:2.0:1761315257.051694:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315257.051697:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315257.051702:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.051730:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322500032:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315257.051742:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000054a8504f x1846869322500224/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315257.051760:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315257.051768:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322499968:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.051789:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eedc2cf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322500416:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315257.051811:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.051824:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315257.051838:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a864682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322500352:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.051855:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322500160:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315257.051868:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.051890:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315257.051908:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500032 00000100:00100000:0.0:1761315257.051913:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322500224:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315257.051938:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315257.051960:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499776 00000100:00100000:0.0:1761315257.051965:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500416 00000100:00100000:0.0:1761315257.052003:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500160 00000100:00100000:0.0:1761315257.052015:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500224 00000100:00100000:0.0:1761315257.052032:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1393 00000100:00100000:0.0:1761315257.052037:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a244fd1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322500416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052080:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a244fd1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322500416:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (257us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.052087:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1393 00000100:00100000:0.0:1761315257.052100:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2045 00000100:00100000:0.0:1761315257.052104:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e401a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322500160:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315257.052187:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315257.052223:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e401a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322500160:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (334us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315257.052230:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2045 00000100:00100000:0.0:1761315257.052244:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2046 00000100:00100000:0.0:1761315257.052248:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee00bcc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322500224:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315257.052338:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315257.052388:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee00bcc3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322500224:12345-0@lo:8:kworker/u8:2.0 Request processed in 138us (449us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315257.052394:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2046 00000100:00080000:0.0:1761315257.052518:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315257.052522:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315257.052525:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322500160:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315257.052534:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315257.052541:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315257.052543:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322500224:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052571:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eedc2cf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322500416:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052600:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499968 00000100:00100000:0.0:1761315257.052630:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499904 00000100:00100000:0.0:1761315257.052648:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2615 00000100:00100000:0.0:1761315257.052651:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322499968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052685:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322499968:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (876us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.052690:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2615 00000100:00100000:0.0:1761315257.052701:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2616 00000100:00100000:0.0:1761315257.052704:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322499904:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315257.052724:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315257.052748:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322499904:12345-0@lo:41:osp-pre-0-1.0 Request processed in 43us (487us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.052753:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2616 00000100:00100000:0.0:1761315257.052770:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322499840:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052791:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.052807:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322500096:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052824:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.052836:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322500480:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052852:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315257.052876:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500096 00000100:00100000:0.0:1761315257.052891:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2617 00000100:00100000:0.0:1761315257.052894:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322500096:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052922:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322500096:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (97us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.052928:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2617 00000100:00100000:0.0:1761315257.052949:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322500096:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052965:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500480 00000100:00100000:0.0:1761315257.052977:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1394 00000100:00100000:0.0:1761315257.052979:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322500480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.052999:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322500480:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (147us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.053002:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1394 00000100:00100000:0.0:1761315257.053015:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322500480:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053029:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500352 00000100:00100000:0.0:1761315257.053042:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500288 00000100:00100000:0.0:1761315257.053048:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322499840 00000100:00100000:0.0:1761315257.053053:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4167 00000100:00100000:0.0:1761315257.053056:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c7ce5eac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322500032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053075:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c7ce5eac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322500032:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (1319us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.053078:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4167 00000100:00100000:0.0:1761315257.053086:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4168 00000100:00100000:0.0:1761315257.053087:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322500352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053101:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322500352:12345-0@lo:400:kworker/u8:2.0 Request processed in 13us (1235us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.053104:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4168 00000100:00100000:0.0:1761315257.053108:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4169 00000100:00100000:0.0:1761315257.053110:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000025edcbcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322500288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053122:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000025edcbcd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322500288:12345-0@lo:400:kworker/u8:2.0 Request processed in 11us (813us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.053125:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4169 00000100:00100000:0.0:1761315257.053129:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4170 00000100:00100000:0.0:1761315257.053131:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba0fa64f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322499840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053144:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba0fa64f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322499840:12345-0@lo:400:kworker/u8:2.0 Request processed in 12us (354us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.053147:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4170 00000100:00100000:0.0:1761315257.053162:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322499840:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053176:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322500032:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053252:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a864682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322500352:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.053260:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322499968:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315257.054488:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1558 00000100:00100000:3.0:1761315257.054496:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322499776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315257.054726:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500544 00000100:00100000:0.0:1761315257.054757:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 929 00000100:00100000:0.0:1761315257.054761:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bbf3014 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322500544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315257.054781:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315257.054832:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bbf3014 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322500544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (2208us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315257.054842:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 929 00000100:00100000:3.0:1761315257.055567:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322499776:12345-0@lo:400:kworker/u8:2.0 Request processed in 1071us (3867us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315257.055576:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1558 00000100:00100000:0.0:1761315257.055701:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322499776:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315259.081865:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315259.082198:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539776 00000100:00100000:3.0:1761315259.082767:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2047 00000100:00100000:3.0:1761315259.082777:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307539776:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315259.082884:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315259.082986:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307539776:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 208us (1134us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315259.082994:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2047 00000100:00100000:2.0:1761315259.084183:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315259.084234:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315259.084363:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539584 00000100:00100000:0.0:1761315259.084424:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1559 00000100:00100000:0.0:1761315259.084431:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307539584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315259.084627:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307539584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 196us (450us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315259.084637:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1559 00000100:00100000:3.0:1761315259.085126:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539712 00000100:00100000:2.0:1761315259.085160:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315259.085219:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539648 00000100:00100000:3.0:1761315259.085260:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4171 00000100:00100000:3.0:1761315259.085268:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307539712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315259.085314:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4172 00000100:00100000:0.0:1761315259.085323:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb727457 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307539648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315259.085415:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb727457 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307539648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 81us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315259.085415:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307539712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 134us (1168us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315259.085445:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4172 00000100:00100000:3.0:1761315259.085595:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4171 00010000:00010000:0.0:1761315259.099512:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315259.613982:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500608 00000100:00100000:0.0:1761315259.614038:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2618 00000100:00100000:0.0:1761315259.614045:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322500608:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315259.614082:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315259.615384:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322500608:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1333us (1525us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315259.615395:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2618 00000100:00100000:2.0:1761315259.616028:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322500608:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315260.123491:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315262.171528:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315262.171535:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315262.171538:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315262.171545:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3504 00000100:00080000:3.0:1761315262.171548:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315262.171553:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315262.171573:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a407033 x1846869322500992/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315262.171585:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315262.171588:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315262.171590:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315262.171593:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3504 00000100:00080000:3.0:1761315262.171594:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315262.171597:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315262.171607:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a5601a5d x1846869322501056/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315262.171906:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322500672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.171994:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315262.172028:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322500928:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.172054:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315262.172062:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500672 00000100:00100000:2.0:1761315262.172068:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322501312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.172092:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315262.172112:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315262.172117:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000535a6e35 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322500864:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315262.172121:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1560 00000100:00100000:3.0:1761315262.172127:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322500672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.172141:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315262.172154:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322501248:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315262.172166:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322500672:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (174us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315262.172171:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315262.172173:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1560 00000100:00100000:2.0:1761315262.172191:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322500672:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315262.172204:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501312 00000100:00100000:2.0:1761315262.172217:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500864 00000100:00100000:3.0:1761315262.172223:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501248 00000100:00100000:3.0:1761315262.172237:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1395 00000100:00100000:2.0:1761315262.172241:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4173 00000100:00100000:3.0:1761315262.172241:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322501312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.172244:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0c9b763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322500864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315262.172287:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322501312:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315262.172294:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1395 00000100:00100000:3.0:1761315262.172307:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1396 00000100:00100000:3.0:1761315262.172311:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322501248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315262.172322:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322500992:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315262.172377:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315262.172391:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5601a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322501056:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315262.172407:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315262.172436:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322500928 00000100:00100000:0.0:1761315262.172457:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2619 00000100:00100000:0.0:1761315262.172460:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322500928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315262.176326:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0c9b763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322500864:12345-0@lo:400:kworker/u8:2.0 Request processed in 4075us (4178us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315262.176355:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4173 00000100:00100000:0.0:1761315262.176729:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322500928:12345-0@lo:400:kworker/u8:2.0 Request processed in 4265us (4672us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315262.176738:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2619 00010000:00010000:0.0:1761315262.176809:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315262.176901:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322501248:12345-0@lo:400:kworker/u8:2.0 Request processed in 4588us (4727us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315262.176911:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1396 00010000:00010000:0.0:1761315263.196042:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315264.201164:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307539840 00000100:00100000:0.0:1761315264.201198:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1561 00000100:00100000:0.0:1761315264.201202:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307539840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315264.201224:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307539840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315264.201229:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1561 00000100:00100000:3.0:1761315264.201924:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540032 00000100:00100000:3.0:1761315264.202423:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2050 00000100:00100000:3.0:1761315264.202429:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307540032:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315264.202521:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315264.202580:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307540032:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 150us (1820us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315264.202587:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2050 00000100:00100000:2.0:1761315264.741729:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501504 00000100:00100000:2.0:1761315264.741786:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2622 00000100:00100000:2.0:1761315264.741792:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfe5acb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322501504:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315264.741825:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315264.741895:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfe5acb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322501504:12345-0@lo:41:osp-pre-1-0.0 Request processed in 103us (242us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315264.741904:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2622 00000100:00100000:0.0:1761315264.741994:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e401a94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322501504:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315265.243532:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315266.267510:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315267.291933:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018b59f1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322501568:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315267.292035:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315267.292132:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315267.292320:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0897acb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322501632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315267.292365:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315267.292442:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501632 00000100:00100000:2.0:1761315267.292489:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 931 00000100:00100000:2.0:1761315267.292496:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9b1c2df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322501632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315267.292523:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315267.292563:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9b1c2df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322501632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315267.292570:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 931 00000100:00100000:2.0:1761315267.292612:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0897acb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322501632:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315267.292912:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018b59f1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322501568:0@lo:41:osp-pre-0-1.0 00000100:00080000:2.0:1761315267.293120:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315267.293125:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315267.293128:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315267.293132:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3509 00000100:00080000:2.0:1761315267.293134:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315267.293138:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315267.293155:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cf11a655 x1846869322501952/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315267.293168:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315267.293171:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315267.293172:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315267.293175:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3509 00000100:00080000:2.0:1761315267.293177:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315267.293179:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315267.293189:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1eea702 x1846869322502016/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315267.293219:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa5147f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322501824:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.293257:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315267.293272:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005bd99502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322502208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.293293:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315267.293308:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315267.293312:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9dd87da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322501760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.293332:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315267.293365:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c0ab56d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322502144:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.293386:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315267.293447:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0db494f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322501888:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315267.293470:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315267.293495:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315267.293500:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0897acb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322501696:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315267.293543:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315267.293558:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322502080:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315267.293578:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315267.293600:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501696 00000100:00100000:0.0:1761315267.293624:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1562 00000100:00100000:0.0:1761315267.293628:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322501696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315267.293660:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322501696:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (118us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315267.293667:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1562 00000100:00100000:0.0:1761315267.293700:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0897acb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322501696:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.293769:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501824 00000100:00100000:3.0:1761315267.293802:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501760 00000100:00100000:3.0:1761315267.293819:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502144 00000100:00100000:3.0:1761315267.293829:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502080 00000100:00100000:3.0:1761315267.293839:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4179 00000100:00100000:3.0:1761315267.293843:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322501824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.293876:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322501824:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (619us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315267.293884:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4179 00000100:00100000:3.0:1761315267.293923:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4180 00000100:00100000:3.0:1761315267.293926:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000381436d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322501760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.293950:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000381436d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322501760:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (618us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315267.293955:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4180 00000100:00100000:3.0:1761315267.293963:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4181 00000100:00100000:3.0:1761315267.293966:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009cdb71e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322502144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.293986:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009cdb71e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322502144:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (600us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315267.293991:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4181 00000100:00100000:3.0:1761315267.294000:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4182 00000100:00100000:3.0:1761315267.294003:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f03c3dae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322502080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294029:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f03c3dae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322502080:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (451us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315267.294034:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4182 00000100:00100000:3.0:1761315267.294058:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322502080:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294079:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9dd87da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322501760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294087:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c0ab56d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322502144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294094:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa5147f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322501824:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294114:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502208 00000100:00100000:3.0:1761315267.294136:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1397 00000100:00100000:3.0:1761315267.294140:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322502208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315267.294168:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322502208:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (875us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315267.294174:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1397 00000100:00100000:3.0:1761315267.294198:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005bd99502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322502208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.294309:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501888 00000100:00100000:2.0:1761315267.294363:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2624 00000100:00100000:2.0:1761315267.294367:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc30b944 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322501888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315267.294408:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc30b944 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322501888:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (937us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315267.294414:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2624 00000100:00100000:0.0:1761315267.294674:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322501952 00000100:00100000:0.0:1761315267.294700:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502016 00000100:00100000:0.0:1761315267.294715:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2051 00000100:00100000:0.0:1761315267.294719:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322501952:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315267.294808:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315267.294839:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322501952:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (298us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315267.294845:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2051 00000100:00100000:0.0:1761315267.294856:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2052 00000100:00100000:0.0:1761315267.294859:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322502016:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315267.294938:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315267.294959:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322502016:12345-0@lo:8:kworker/u8:0.0 Request processed in 99us (373us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315267.294964:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2052 00000100:00100000:3.0:1761315267.295314:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0db494f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322501888:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315268.315880:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315269.325026:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540224 00000100:00100000:2.0:1761315269.325080:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540160 00000100:00100000:2.0:1761315269.325106:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4183 00000100:00100000:2.0:1761315269.325113:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a2c7705 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307540224:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315269.325178:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a2c7705 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307540224:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 64us (198us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315269.325185:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4183 00000100:00100000:2.0:1761315269.325198:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4184 00000100:00100000:2.0:1761315269.325201:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1adc0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307540160:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315269.325217:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1adc0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307540160:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 15us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315269.325223:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4184 00000100:00100000:0.0:1761315269.334396:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540096 00000100:00100000:0.0:1761315269.334457:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1563 00000100:00100000:0.0:1761315269.334464:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307540096:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315269.334537:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307540096:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 73us (9606us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315269.334543:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1563 00010000:00010000:3.0:1761315269.339513:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315269.851594:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322502272:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315269.851665:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315269.851827:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502272 00000100:00100000:0.0:1761315269.851881:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2625 00000100:00100000:0.0:1761315269.851887:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322502272:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315269.851918:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315269.851967:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322502272:12345-0@lo:41:osp-pre-1-0.0 Request processed in 80us (303us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315269.851973:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2625 00000100:00100000:3.0:1761315269.852094:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322502272:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315270.364889:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315271.387500:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315272.411571:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322502336:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315272.411640:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315272.411671:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322502400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315272.411700:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315272.412253:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315272.412259:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315272.412262:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761315272.412262:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322502528:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315272.412269:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3514 00000100:00080000:0.0:1761315272.412272:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315272.412277:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315272.412297:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a06c7ae2 x1846869322502784/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315272.412306:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315272.412319:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315272.412322:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315272.412323:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315272.412327:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3514 00000100:00080000:0.0:1761315272.412328:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315272.412331:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315272.412334:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315272.412366:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000064ec7a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322502464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412368:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000292a2071 x1846869322502848/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315272.412400:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315272.412420:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322502720:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.412442:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315272.412458:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315272.412466:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000509695a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322502912:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412476:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502528 00000100:00100000:2.0:1761315272.412490:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315272.412517:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502720 00000100:00100000:2.0:1761315272.412536:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2626 00000100:00100000:2.0:1761315272.412542:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322502528:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412549:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2627 00000100:00100000:0.0:1761315272.412554:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322502720:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412727:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322502720:12345-0@lo:400:kworker/u8:2.0 Request processed in 171us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315272.412735:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2627 00000100:00100000:0.0:1761315272.412759:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502464 00000100:00100000:0.0:1761315272.412782:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502912 00000100:00100000:0.0:1761315272.412799:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4185 00000100:00100000:0.0:1761315272.412802:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322502464:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412830:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322502464:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (431us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315272.412836:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4185 00000100:00100000:0.0:1761315272.412848:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4186 00000100:00100000:0.0:1761315272.412852:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322502912:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412875:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322502912:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (386us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315272.412880:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4186 00000100:00100000:0.0:1761315272.412933:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000509695a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322502912:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.412948:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000064ec7a9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322502464:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.412955:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322502528:12345-0@lo:400:kworker/u8:2.0 Request processed in 411us (652us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315272.412960:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322502720:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.412966:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2626 00000100:00100000:0.0:1761315272.412974:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322502528:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413008:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322502592:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.413022:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502400 00000100:00100000:2.0:1761315272.413040:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315272.413049:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1564 00000100:00100000:0.0:1761315272.413054:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322502400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413057:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007349fc1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322502976:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413092:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315272.413100:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322502400:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (1399us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315272.413108:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1564 00000100:00100000:2.0:1761315272.413113:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315272.413121:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010aaa306 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322502656:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315272.413145:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315272.413152:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a06c7ae2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322502784:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413161:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322503040:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.413186:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315272.413201:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502592 00000100:00100000:0.0:1761315272.413234:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322502848:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413239:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502976 00000100:00100000:0.0:1761315272.413260:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315272.413264:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4187 00000100:00100000:2.0:1761315272.413268:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd99502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322502592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.413300:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502784 00000100:00100000:2.0:1761315272.413306:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd99502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322502592:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315272.413313:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4187 00000100:00100000:0.0:1761315272.413320:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502848 00000100:00100000:2.0:1761315272.413329:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4188 00000100:00100000:2.0:1761315272.413335:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5147f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322502976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315272.413340:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2054 00000100:00100000:0.0:1761315272.413372:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322502784:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315272.413403:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5147f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322502976:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (311us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315272.413409:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4188 00000100:00100000:2.0:1761315272.413431:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502656 00010000:02020000:0.0:1761315272.413460:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315272.413492:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322502784:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (308us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315272.413498:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2054 00000100:00100000:0.0:1761315272.413509:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2055 00000100:00100000:0.0:1761315272.413512:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322502848:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315272.414968:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2628 00000100:00100000:2.0:1761315272.414974:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322502656:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315272.415003:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315272.415048:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322502656:12345-0@lo:41:osp-pre-0-1.0 Request processed in 72us (1902us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315272.415059:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2628 00000100:00100000:2.0:1761315272.415104:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322502336 00010000:02020000:0.0:1761315272.415233:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315272.415269:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322502848:12345-0@lo:8:kworker/u8:2.0 Request processed in 1756us (2011us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315272.415275:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2055 00000100:00080000:0.0:1761315272.415301:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315272.415304:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315272.415307:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000292a2071 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322502848:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315272.415380:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322502400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315272.415510:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322502336:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315273.436281:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315274.440401:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540352 00000100:00100000:0.0:1761315274.440438:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1565 00000100:00100000:0.0:1761315274.440442:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307540352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315274.440510:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307540352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (382us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315274.440523:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1565 00000100:00100000:3.0:1761315274.440661:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540480 00000100:00100000:3.0:1761315274.440694:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540416 00000100:00100000:3.0:1761315274.440719:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4189 00000100:00100000:3.0:1761315274.440723:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010aaa306 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307540480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315274.440753:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010aaa306 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307540480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 30us (565us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315274.440759:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4189 00000100:00100000:3.0:1761315274.440770:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4190 00000100:00100000:3.0:1761315274.440773:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df87a9cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307540416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315274.440790:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df87a9cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307540416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 16us (525us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315274.440795:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4190 00010000:00010000:0.0:1761315274.460119:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315274.973294:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503104 00000100:00100000:3.0:1761315274.973365:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2629 00000100:00100000:3.0:1761315274.973372:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322503104:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315274.973401:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315274.973449:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322503104:12345-0@lo:41:osp-pre-1-0.0 Request processed in 77us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315274.973455:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2629 00010000:00010000:0.0:1761315275.485708:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315276.507494:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315277.531555:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315277.531562:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315277.531565:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315277.531573:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3519 00000100:00080000:3.0:1761315277.531576:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315277.531582:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315277.531601:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000078a92c3a x1846869322503488/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315277.531614:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315277.531616:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315277.531617:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315277.531620:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3519 00000100:00080000:3.0:1761315277.531622:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315277.531624:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315277.531635:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d2a9d0ea x1846869322503552/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315277.531731:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322503360:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.531790:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.531813:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d7c0a8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322503744:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.531836:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.531854:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:2.0:1761315277.531859:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322503168:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.531901:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.531932:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000358802ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322503424:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.531957:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.531971:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006be81110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322503808:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532003:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.532034:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503168 00000100:00100000:2.0:1761315277.532072:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1566 00000100:00100000:0.0:1761315277.532080:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322503488:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532082:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322503168:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315277.532129:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.532132:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322503168:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (229us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315277.532141:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1566 00000100:00100000:0.0:1761315277.532152:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2a9d0ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322503552:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315277.532207:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315277.532237:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503424 00000100:00100000:0.0:1761315277.532271:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2630 00000100:00100000:0.0:1761315277.532277:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322503424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532306:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322503232:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315277.532310:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322503424:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (352us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315277.532317:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2630 00000100:00100000:0.0:1761315277.532372:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2057 00000100:00100000:0.0:1761315277.532377:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064ec7a9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322503488:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532388:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.532410:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a293e837 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322503616:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532435:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315277.532440:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322503296:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532452:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018b59f1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322503872:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315277.532477:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.532487:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315277.532496:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ab44e19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322503680:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315277.532531:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315277.532545:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503872 00000100:00100000:3.0:1761315277.532577:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503232 00000100:00100000:2.0:1761315277.532581:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 933 00000100:00100000:2.0:1761315277.532586:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eedc2cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322503872:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315277.532609:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503616 00002000:00000024:2.0:1761315277.532610:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00010000:02020000:0.0:1761315277.532629:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315277.532633:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503680 00000100:00100000:2.0:1761315277.532650:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eedc2cf8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322503872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (163us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315277.532660:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4192 00000100:00100000:2.0:1761315277.532665:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 933 00000100:00100000:3.0:1761315277.532669:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322503232:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315277.532674:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064ec7a9a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322503488:12345-0@lo:8:kworker/u8:2.0 Request processed in 293us (546us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315277.532684:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2057 00000100:00100000:2.0:1761315277.532709:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4193 00000100:00100000:3.0:1761315277.532718:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322503232:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315277.532720:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4194 00000100:00100000:2.0:1761315277.532720:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322503616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315277.532729:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081d868c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322503680:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315277.532734:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4192 00000100:00100000:2.0:1761315277.532771:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322503616:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (334us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315277.532778:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081d868c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322503680:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (249us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315277.532782:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503296 00000100:00100000:2.0:1761315277.532783:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4193 00000100:00100000:0.0:1761315277.532791:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4194 00000100:00100000:3.0:1761315277.532817:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2631 00000100:00100000:3.0:1761315277.532826:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322503296:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532830:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322503232:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532852:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a293e837 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322503616:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315277.532861:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315277.532872:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018b59f1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322503872:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315277.532875:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322503296:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (397us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315277.532884:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2631 00000100:00100000:2.0:1761315277.532894:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315277.532904:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000075457230 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322503936:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315277.532918:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322503296:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315277.532930:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315277.532932:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ab44e19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322503680:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315277.533230:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322503936 00000100:00100000:0.0:1761315277.533261:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2632 00000100:00100000:0.0:1761315277.533264:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322503936:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315277.533283:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315277.533321:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322503936:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315277.533329:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2632 00010000:00010000:0.0:1761315278.555528:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315279.562151:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315279.562212:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315279.562254:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315279.562288:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315279.562320:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540736 00000100:00100000:2.0:1761315279.562382:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540672 00000100:00100000:2.0:1761315279.562406:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4195 00000100:00100000:2.0:1761315279.562412:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018b59f1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307540736:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315279.562462:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018b59f1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307540736:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 49us (208us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315279.562470:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4195 00000100:00100000:2.0:1761315279.562488:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4196 00000100:00100000:2.0:1761315279.562492:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307540672:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315279.562516:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307540672:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 23us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315279.562522:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4196 00000100:00100000:0.0:1761315279.562653:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540800 00000100:00100000:0.0:1761315279.562692:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2059 00000100:00100000:0.0:1761315279.562698:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307540800:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315279.562795:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315279.562844:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307540800:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 146us (701us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315279.562852:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2059 00010000:00010000:0.0:1761315279.579626:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315280.091673:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504000 00000100:00100000:2.0:1761315280.091966:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2633 00000100:00100000:2.0:1761315280.091972:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075457230 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322504000:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315280.092005:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315280.092057:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075457230 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322504000:12345-0@lo:41:osp-pre-1-0.0 Request processed in 84us (438us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315280.092065:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2633 00010000:00010000:0.0:1761315280.603512:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315281.628336:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315282.651494:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315282.651525:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315282.651544:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322504064:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315282.651605:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315282.651664:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504064 00000100:00100000:3.0:1761315282.651721:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2634 00000100:00100000:3.0:1761315282.651729:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbe04555 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322504064:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315282.651762:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:2.0:1761315282.651799:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315282.651804:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315282.651808:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315282.651809:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbe04555 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322504064:12345-0@lo:41:osp-pre-0-1.0 Request processed in 79us (203us total) trans 0 rc 0/0 00000100:00080000:2.0:1761315282.651816:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3525 00000100:00080000:2.0:1761315282.651820:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315282.651825:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2634 00000100:00080000:2.0:1761315282.651826:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315282.651846:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000443cb307 x1846869322504448/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315282.651854:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322504320:0@lo:400:kworker/u8:1.0 00000100:00080000:2.0:1761315282.651865:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315282.651870:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315282.651871:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315282.651876:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3525 00000100:00080000:2.0:1761315282.651878:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315282.651881:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315282.651896:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b7b15fb6 x1846869322504512/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315282.651901:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315282.651928:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315282.651938:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5601a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322504256:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315282.651971:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322504448:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315282.651993:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315282.652010:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315282.652025:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322504640:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315282.652035:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322504512:0@lo:8:kworker/u8:1.0 00000100:00100000:2.0:1761315282.652075:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315282.652075:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315282.652115:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322504704:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315282.652121:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504768 00000100:00100000:2.0:1761315282.652166:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504448 00000100:00100000:3.0:1761315282.652180:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315282.652195:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504512 00000100:00100000:2.0:1761315282.652206:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504704 00000100:00100000:2.0:1761315282.652228:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1401 00000100:00100000:3.0:1761315282.652229:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504320 00000100:00100000:2.0:1761315282.652234:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322504768:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652247:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1568 00000100:00100000:3.0:1761315282.652266:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504256 00000100:00100000:2.0:1761315282.652279:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322504768:12345-0@lo:400:kworker/u8:1.0 Request processed in 44us (292us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315282.652283:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1402 00000100:00100000:1.0:1761315282.652291:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322504704:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652293:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504640 00000100:00100000:3.0:1761315282.652314:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504576 00000100:00100000:1.0:1761315282.652329:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322504704:12345-0@lo:400:kworker/u8:1.0 Request processed in 37us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652334:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4197 00000100:00100000:1.0:1761315282.652338:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1402 00000100:00100000:3.0:1761315282.652365:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322504320:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652371:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2060 00000100:00100000:1.0:1761315282.652375:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322504448:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652405:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322504320:12345-0@lo:400:kworker/u8:1.0 Request processed in 62us (509us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652412:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4197 00000100:00100000:3.0:1761315282.652426:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4198 00000100:00100000:3.0:1761315282.652430:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322504256:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652458:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322504256:12345-0@lo:400:kworker/u8:1.0 Request processed in 27us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652463:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4198 00010000:02020000:1.0:1761315282.652465:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315282.652478:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4199 00000100:00100000:3.0:1761315282.652482:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322504640:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652502:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322504448:12345-0@lo:8:kworker/u8:1.0 Request processed in 124us (494us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315282.652509:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2060 00000100:00100000:3.0:1761315282.652521:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322504640:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (453us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315282.652526:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2061 00000100:00100000:1.0:1761315282.652536:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322504512:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652536:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4199 00000100:00100000:3.0:1761315282.652556:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4200 00000100:00100000:3.0:1761315282.652560:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006be81110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322504576:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652582:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006be81110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322504576:12345-0@lo:400:kworker/u8:1.0 Request processed in 21us (475us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652587:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4200 00010000:02020000:1.0:1761315282.652613:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315282.652616:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504384 00000100:00100000:3.0:1761315282.652642:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2635 00000100:00100000:1.0:1761315282.652645:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322504512:12345-0@lo:8:kworker/u8:1.0 Request processed in 111us (576us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315282.652650:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a293e837 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322504384:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652656:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2061 00000100:00100000:1.0:1761315282.652679:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504192 00000100:00100000:3.0:1761315282.652687:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a293e837 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322504384:12345-0@lo:400:kworker/u8:1.0 Request processed in 38us (762us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652694:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2635 00000100:00100000:1.0:1761315282.652707:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 934 00000100:00100000:1.0:1761315282.652711:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d7c0a8d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322504192:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315282.652730:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315282.652735:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322504704:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652751:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5601a5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322504256:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652763:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d7c0a8d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322504192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (707us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315282.652767:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322504640:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652772:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 934 00000100:00100000:3.0:1761315282.652783:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322504320:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315282.652833:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322504064:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761315282.652869:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315282.652872:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ab44e19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322504192:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761315282.652874:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315282.652881:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322504448:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652893:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000741fc930 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322504576:0@lo:400:kworker/u8:1.0 00000100:00080000:1.0:1761315282.652896:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315282.652900:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315282.652905:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322504512:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652913:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322504768:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652926:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000535a6e35 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322504128:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315282.652937:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322504384:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315282.654290:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1401 00010000:00010000:0.0:1761315283.675763:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315284.692614:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315284.692681:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315284.692687:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540928 00000100:00100000:1.0:1761315284.692738:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4201 00000100:00100000:1.0:1761315284.692745:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307540928:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315284.692807:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307540928:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 62us (202us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315284.692813:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4201 00000100:00100000:0.0:1761315284.694026:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541056 00000100:00100000:0.0:1761315284.694074:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2062 00000100:00100000:0.0:1761315284.694081:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307541056:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315284.694181:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315284.694253:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307541056:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 171us (1572us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315284.694261:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2062 00000100:00100000:1.0:1761315284.696813:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315284.696866:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315284.696992:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540864 00000100:00100000:2.0:1761315284.697042:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1569 00000100:00100000:2.0:1761315284.697047:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307540864:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315284.697115:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307540864:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 67us (309us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315284.697131:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1569 00000100:00100000:3.0:1761315284.697994:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307540992 00000100:00100000:1.0:1761315284.698936:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4202 00000100:00100000:1.0:1761315284.698945:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ab44e19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307540992:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315284.699024:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ab44e19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307540992:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 78us (2158us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315284.699032:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4202 00010000:00010000:0.0:1761315284.699451:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315285.211773:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322504832:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315285.211872:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315285.212515:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504832 00000100:00100000:1.0:1761315285.212571:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2636 00000100:00100000:1.0:1761315285.212576:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a49b2fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322504832:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315285.212609:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315285.212665:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a49b2fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322504832:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (794us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315285.212672:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2636 00000100:00100000:3.0:1761315285.212730:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0758582 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322504832:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315285.723789:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315286.747621:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315287.771510:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f4362ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322504896:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315287.771604:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315287.771616:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315287.771653:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504896 00000100:00080000:3.0:1761315287.772185:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315287.772191:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315287.772194:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315287.772199:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3530 00000100:00080000:3.0:1761315287.772202:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315287.772207:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315287.772224:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d26c929f x1846869322505216/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315287.772238:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315287.772242:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315287.772244:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315287.772248:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3530 00000100:00080000:3.0:1761315287.772249:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:1.0:1761315287.772253:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2637 00000100:00080000:3.0:1761315287.772254:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315287.772264:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f903cab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322504896:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315287.772268:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001883266f x1846869322505280/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000004:00000024:1.0:1761315287.772301:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315287.772316:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c5722f38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322504960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.772567:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322505152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.772661:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322505024:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315287.772686:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315287.772710:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a83abb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322505344:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315287.772740:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322505216:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315287.772765:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315287.772780:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322505280:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315287.772798:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315287.772824:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322504960 00000100:00100000:3.0:1761315287.772850:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1570 00000100:00100000:3.0:1761315287.772855:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322504960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315287.772883:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322504960:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (199us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315287.772890:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1570 00000100:00100000:3.0:1761315287.772917:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505216 00000100:00100000:3.0:1761315287.772931:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505280 00000100:00100000:3.0:1761315287.772943:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2063 00000100:00100000:3.0:1761315287.772946:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322505216:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315287.773491:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315287.773518:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024202054 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322505408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.773545:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315287.773575:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315287.773582:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d164467 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322505088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.773607:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315287.773620:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f903cab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322504896:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1355us (2003us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315287.773628:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505472:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315287.773630:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2637 00000100:00100000:2.0:1761315287.773659:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315287.773672:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505472 00000100:00100000:2.0:1761315287.773695:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505024 00000100:00100000:1.0:1761315287.773706:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1403 00000100:00100000:1.0:1761315287.773711:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322505472:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761315287.773714:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315287.773762:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322505472:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (105us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315287.773772:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1403 00000100:00100000:3.0:1761315287.773772:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322505216:12345-0@lo:8:kworker/u8:2.0 Request processed in 820us (1003us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315287.773783:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2063 00000100:00100000:1.0:1761315287.773788:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2064 00000100:00100000:1.0:1761315287.773792:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322505280:12345-0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315287.773821:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315287.773825:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315287.773830:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322505216:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315287.773855:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505408 00010000:02020000:1.0:1761315287.773876:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315287.773879:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505088 00000100:00100000:3.0:1761315287.773901:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4203 00000100:00100000:3.0:1761315287.773906:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322505408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315287.773915:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322505280:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (1114us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315287.773922:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2064 00000100:00100000:3.0:1761315287.773946:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322505408:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (399us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315287.773952:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4204 00000100:00100000:3.0:1761315287.773960:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4203 00000100:00100000:1.0:1761315287.773961:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322505088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315287.773997:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322505088:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (391us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315287.774003:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4204 00000100:00100000:0.0:1761315287.774114:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315287.774143:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039d27018 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322505536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.774161:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4205 00000100:00100000:2.0:1761315287.774167:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322505024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.774174:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315287.774212:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f4362ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322504896:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315287.774215:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322505024:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (724us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315287.774225:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4205 00000100:00100000:0.0:1761315287.774232:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315287.774241:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322505600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315287.774264:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d164467 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322505088:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.774273:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315287.774280:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.774298:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322505024:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.774308:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505536 00000100:00100000:2.0:1761315287.774313:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024202054 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322505408:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.774339:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1404 00000100:00100000:0.0:1761315287.774372:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322505536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.774406:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322505536:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (235us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315287.774412:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1404 00000100:00100000:0.0:1761315287.774431:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505600 00000100:00100000:0.0:1761315287.774452:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 935 00000100:00100000:0.0:1761315287.774456:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e401a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322505600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315287.774476:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315287.774514:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e401a94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322505600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 57us (243us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315287.774523:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 935 00000100:00100000:0.0:1761315287.774560:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322505600:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315287.774571:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039d27018 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322505536:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315287.774803:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505152 00000100:00100000:2.0:1761315287.774832:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315287.774839:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2638 00000100:00100000:1.0:1761315287.774844:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3c2a916 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322505152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.774866:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c5722f38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322504960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315287.774887:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3c2a916 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322505152:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (772us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315287.774893:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505344 00000100:00100000:1.0:1761315287.774894:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2638 00000100:00100000:2.0:1761315287.774923:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4206 00000100:00100000:2.0:1761315287.774928:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322505344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315287.774972:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322505344:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (142us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315287.774979:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4206 00000100:00080000:3.0:1761315287.775076:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315287.775081:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315287.775085:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322505280:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315287.775846:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a83abb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322505344:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315287.775950:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322505152:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315288.795512:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315289.800580:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315289.800629:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315289.800642:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541120 00000100:00100000:1.0:1761315289.800668:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315289.800701:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1571 00000100:00100000:3.0:1761315289.800708:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307541120:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315289.800778:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307541120:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 68us (206us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315289.800785:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541184 00000100:00100000:3.0:1761315289.800785:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1571 00000100:00100000:0.0:1761315289.800826:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541248 00000100:00100000:3.0:1761315289.800830:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4207 00000100:00100000:3.0:1761315289.800835:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052159ae1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307541184:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315289.800854:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4208 00000100:00100000:0.0:1761315289.800861:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000561c506d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307541248:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315289.800868:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052159ae1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307541184:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 31us (240us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315289.800875:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4207 00000100:00100000:0.0:1761315289.800908:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000561c506d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307541248:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 46us (240us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315289.800916:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4208 00000100:00100000:2.0:1761315289.801929:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315289.803385:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541312 00000100:00100000:0.0:1761315289.804178:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2065 00000100:00100000:0.0:1761315289.804185:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307541312:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315289.804279:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315289.804371:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307541312:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 184us (2446us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315289.804379:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2065 00010000:00010000:0.0:1761315289.820436:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315290.331661:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315290.331724:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322505664:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315290.331846:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315290.331940:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505664 00000100:00100000:0.0:1761315290.331992:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2639 00000100:00100000:0.0:1761315290.331998:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015b27f3c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322505664:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315290.332031:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315290.332105:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015b27f3c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322505664:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315290.332114:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2639 00000100:00100000:3.0:1761315290.332243:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322505664:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315290.843884:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315291.867716:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315292.891623:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315292.891766:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315292.891779:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be6b3021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505728:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315292.891841:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315292.892476:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315292.892480:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315292.892484:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315292.892490:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3535 00000100:00080000:3.0:1761315292.892492:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315292.892497:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315292.892514:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009f48f386 x1846869322506112/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315292.892527:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315292.892530:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315292.892532:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315292.892535:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3535 00000100:00080000:3.0:1761315292.892538:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315292.892541:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315292.892553:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000038a955f1 x1846869322506176/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315292.892576:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505920:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.892599:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322505792:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.892609:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315292.892635:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505728 00000100:00100000:3.0:1761315292.892640:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315292.892659:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b04bb871 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506048:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315292.892684:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2640 00000100:00100000:3.0:1761315292.892688:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f48f386 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322506112:0@lo:8:kworker/u8:1.0 00000100:00100000:1.0:1761315292.892693:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0758582 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322505728:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315292.892716:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:1.0:1761315292.892727:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315292.892735:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038a955f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322506176:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315292.892769:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315292.892776:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0758582 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322505728:12345-0@lo:41:osp-pre-0-1.0 Request processed in 84us (935us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315292.892784:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2640 00000100:00100000:3.0:1761315292.892806:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505792 00000100:00100000:1.0:1761315292.892810:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322505856:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.892838:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1572 00000100:00100000:1.0:1761315292.892842:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315292.892846:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322505792:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315292.892861:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322506240:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315292.892890:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315292.892900:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322505792:12345-0@lo:400:kworker/u8:1.0 Request processed in 55us (260us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315292.892907:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1572 00000100:00100000:1.0:1761315292.892927:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506112 00000100:00100000:3.0:1761315292.892949:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322505984:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315292.892950:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506176 00000100:00100000:1.0:1761315292.892969:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2066 00000100:00100000:1.0:1761315292.892974:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322506112:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315292.892981:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315292.892998:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322506368:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.893017:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315292.893032:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315292.893037:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322506304:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.893055:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315292.893091:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315292.893095:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506368 00000100:00100000:3.0:1761315292.893122:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1405 00000100:00100000:3.0:1761315292.893137:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322506368:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.893172:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322506368:12345-0@lo:400:kworker/u8:1.0 Request processed in 45us (154us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315292.893177:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1405 00000100:00100000:3.0:1761315292.893189:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2067 00000100:00100000:3.0:1761315292.893192:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322506176:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:3.0:1761315292.893270:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315292.894193:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be6b3021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505728:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315292.894237:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505920 00000100:00100000:0.0:1761315292.894267:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2641 00000100:00100000:0.0:1761315292.894272:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbe04555 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322505920:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894314:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbe04555 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322505920:12345-0@lo:400:kworker/u8:1.0 Request processed in 42us (1706us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894321:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2641 00000100:00100000:0.0:1761315292.894364:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505856 00000100:00100000:0.0:1761315292.894389:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506240 00000100:00100000:0.0:1761315292.894404:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322505984 00000100:00100000:0.0:1761315292.894414:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506304 00000100:00100000:0.0:1761315292.894425:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4209 00000100:00100000:0.0:1761315292.894429:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093ab9157 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322505856:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894459:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093ab9157 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322505856:12345-0@lo:400:kworker/u8:1.0 Request processed in 29us (1619us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894464:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4209 00000100:00100000:0.0:1761315292.894474:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4210 00000100:00100000:0.0:1761315292.894477:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322506240:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894498:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b6179b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322506240:12345-0@lo:400:kworker/u8:1.0 Request processed in 20us (1609us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894502:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4210 00000100:00100000:0.0:1761315292.894510:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4211 00000100:00100000:0.0:1761315292.894513:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5722f38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322505984:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894530:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5722f38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322505984:12345-0@lo:400:kworker/u8:1.0 Request processed in 17us (1555us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894535:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4211 00000100:00100000:0.0:1761315292.894542:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4212 00000100:00100000:0.0:1761315292.894545:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322506304:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894562:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322506304:12345-0@lo:400:kworker/u8:1.0 Request processed in 16us (1507us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894567:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4212 00000100:00100000:0.0:1761315292.894596:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322505920:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894669:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322506304:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894678:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322505984:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894686:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322506368:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894723:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b86079b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322506496:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315292.894748:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315292.894768:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322505856:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894776:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322506240:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894802:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506496 00000100:00100000:0.0:1761315292.894821:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 936 00000100:00100000:0.0:1761315292.894825:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000975475fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322506496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315292.894843:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315292.894871:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000975475fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322506496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (123us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.894877:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 936 00000100:00100000:0.0:1761315292.894901:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b86079b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322506496:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315292.894923:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315292.894938:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506432:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.894957:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315292.894981:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322505792:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315292.895002:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322506176:12345-0@lo:8:kworker/u8:1.0 Request processed in 1806us (2231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315292.895003:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506432 00000100:00100000:3.0:1761315292.895013:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2067 00000100:00100000:0.0:1761315292.895028:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1406 00000100:00100000:0.0:1761315292.895034:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d148ece5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506432:12345-0@lo:400:kworker/u8:1.0 00000100:00080000:3.0:1761315292.895056:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315292.895070:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315292.895075:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038a955f1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322506176:0@lo:8:kworker/u8:1.0 00000100:00100000:0.0:1761315292.895082:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d148ece5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506432:12345-0@lo:400:kworker/u8:1.0 Request processed in 46us (122us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.895089:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1406 00000100:00100000:0.0:1761315292.895117:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506432:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.895133:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506048 00000100:00100000:0.0:1761315292.895167:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2642 00000100:00100000:0.0:1761315292.895170:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000292a2071 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506048:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315292.895197:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000292a2071 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506048:12345-0@lo:400:kworker/u8:1.0 Request processed in 26us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315292.895204:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2642 00000100:00100000:0.0:1761315292.895223:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b04bb871 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506048:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315292.895369:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322506112:12345-0@lo:8:kworker/u8:1.0 Request processed in 2394us (2653us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315292.895376:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2066 00000100:00080000:2.0:1761315292.895440:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315292.895443:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315292.895447:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f48f386 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322506112:0@lo:8:kworker/u8:1.0 00010000:00010000:0.0:1761315293.917445:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315294.920113:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315294.920164:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315294.920195:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315294.920280:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315294.920320:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541568 00000100:00100000:1.0:1761315294.920388:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2068 00000100:00100000:1.0:1761315294.920395:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f4362ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307541568:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:1.0:1761315294.920486:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315294.920545:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f4362ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307541568:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 149us (265us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315294.920552:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2068 00000100:00100000:0.0:1761315294.920695:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541440 00000100:00100000:0.0:1761315294.920736:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541504 00000100:00100000:0.0:1761315294.920758:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4213 00000100:00100000:0.0:1761315294.920762:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a293e837 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307541440:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315294.920805:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a293e837 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307541440:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 41us (641us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315294.920811:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4213 00000100:00100000:0.0:1761315294.920824:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4214 00000100:00100000:0.0:1761315294.920828:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be6b3021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307541504:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315294.920852:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be6b3021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307541504:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 24us (658us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315294.920858:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4214 00000100:00100000:0.0:1761315294.923445:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541376 00000100:00100000:0.0:1761315294.923499:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1573 00000100:00100000:0.0:1761315294.923504:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541376:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315294.923564:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541376:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 60us (3459us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315294.923571:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1573 00010000:00010000:0.0:1761315294.939491:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315295.451625:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322506560:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315295.451693:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315295.451797:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506560 00000100:00100000:0.0:1761315295.451846:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2643 00000100:00100000:0.0:1761315295.451852:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322506560:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315295.451881:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315295.451922:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322506560:12345-0@lo:41:osp-pre-1-0.0 Request processed in 70us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315295.451928:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2643 00000100:00100000:2.0:1761315295.452275:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322506560:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315295.963540:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315296.987529:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315298.011697:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315298.011703:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322506752:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315298.011706:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315298.011711:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315298.011718:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3540 00000100:00100000:3.0:1761315298.011718:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007349fc1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322506624:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315298.011722:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315298.011730:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315298.011752:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e57767e2 x1846869322507008/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315298.011768:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315298.011771:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315298.011773:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315298.011779:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3540 00000100:00100000:2.0:1761315298.011779:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315298.011784:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315298.011784:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315298.011789:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315298.011801:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000738cd15f x1846869322507072/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315298.011815:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322506880:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.011823:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f31517f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322507136:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315298.011849:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315298.011868:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322506688:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315298.011895:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322507264:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.011907:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315298.011956:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a06c7ae2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322506816:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.011990:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506944:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315298.011999:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315298.012049:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315298.012056:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507264 00000100:00100000:2.0:1761315298.012070:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000349be38c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322507328:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.012098:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315298.012104:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1407 00000100:00100000:2.0:1761315298.012105:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315298.012113:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322507264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.012130:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc01d0b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322507200:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.012133:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315298.012141:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322507392:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315298.012167:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315298.012174:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315298.012211:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e57767e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322507008:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315298.012214:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506752 00000100:00100000:2.0:1761315298.012238:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315298.012246:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506880 00000100:00100000:2.0:1761315298.012257:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322507072:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315298.012270:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507200 00000100:00100000:2.0:1761315298.012282:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315298.012295:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4215 00000100:00100000:1.0:1761315298.012300:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322506752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.012317:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4216 00000100:00100000:2.0:1761315298.012322:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038a955f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322506880:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.012338:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322506752:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (562us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315298.012379:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4215 00000100:00100000:3.0:1761315298.012441:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322507264:12345-0@lo:400:kworker/u8:2.0 Request processed in 328us (442us total) trans 0 rc 0/0 00010000:00010000:0.0:1761315298.012467:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315298.012494:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4217 00000100:00100000:0.0:1761315298.012498:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f170fef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322507200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.012640:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038a955f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322506880:12345-0@lo:400:kworker/u8:2.0 Request processed in 307us (783us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315298.012648:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4216 00000100:00100000:2.0:1761315298.012672:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506688 00000100:00100000:2.0:1761315298.012698:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 937 00000100:00100000:2.0:1761315298.012701:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322506688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315298.012722:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315298.012756:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322506688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (852us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315298.012763:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 937 00000100:00100000:2.0:1761315298.012782:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507328 00000100:00100000:1.0:1761315298.012962:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507008 00000100:00100000:0.0:1761315298.013036:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f170fef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322507200:12345-0@lo:400:kworker/u8:2.0 Request processed in 536us (870us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315298.013043:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4217 00000100:00100000:0.0:1761315298.013065:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506944 00000100:00100000:0.0:1761315298.013088:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506816 00000100:00100000:0.0:1761315298.013098:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507392 00000100:00100000:0.0:1761315298.013109:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2644 00000100:00100000:0.0:1761315298.013113:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315298.013139:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322506944:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1120us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315298.013145:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2644 00000100:00100000:0.0:1761315298.013154:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2645 00000100:00100000:0.0:1761315298.013157:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322506816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315298.013429:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1407 00000100:00100000:3.0:1761315298.013453:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507072 00000100:00100000:1.0:1761315298.013458:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1408 00000100:00100000:1.0:1761315298.013466:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322507328:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315298.013471:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2069 00000100:00100000:3.0:1761315298.013476:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322507008:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315298.013517:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322507328:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (1402us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315298.013524:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1408 00000100:00100000:1.0:1761315298.013537:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2070 00000100:00100000:1.0:1761315298.013540:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091fbe2e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322507072:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315298.013622:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315298.013658:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091fbe2e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322507072:12345-0@lo:8:kworker/u8:2.0 Request processed in 116us (1376us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315298.013664:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2070 00000100:00100000:1.0:1761315298.013684:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2646 00000100:00100000:1.0:1761315298.013688:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030335973 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322507392:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315298.013710:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315298.013742:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030335973 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322507392:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (1569us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315298.013748:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2646 00000100:00100000:2.0:1761315298.013765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322507392:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315298.013785:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322506688:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761315298.013788:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315298.013793:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315298.013799:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322507072:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315298.013801:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322506944:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.013815:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000349be38c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322507328:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.013842:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc01d0b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322507200:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.013887:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315298.013918:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322507264:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.013923:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095e7dbc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322506752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.013936:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041264762 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322506880:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.013957:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322506624 00000100:00100000:1.0:1761315298.013981:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507136 00000100:00100000:2.0:1761315298.013986:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1574 00000100:00100000:2.0:1761315298.013991:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322506624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.014012:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4218 00000100:00100000:1.0:1761315298.014018:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfe5acb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322507136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315298.014048:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322506624:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (2269us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315298.014056:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1574 00000100:00100000:1.0:1761315298.014076:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfe5acb4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322507136:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315298.014083:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4218 00000100:00100000:2.0:1761315298.014111:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f31517f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322507136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315298.014120:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007349fc1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322506624:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315298.014312:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322506816:12345-0@lo:400:kworker/u8:2.0 Request processed in 1153us (2217us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315298.014319:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2645 00000100:00100000:2.0:1761315298.014987:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a06c7ae2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322506816:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761315298.015513:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315298.015573:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322507008:12345-0@lo:8:kworker/u8:2.0 Request processed in 2095us (3336us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315298.015581:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2069 00000100:00080000:2.0:1761315298.015635:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315298.015640:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315298.015645:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e57767e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322507008:0@lo:8:kworker/u8:2.0 00010000:00010000:2.0:1761315299.043657:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315300.040974:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315300.042293:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541696 00000100:00100000:1.0:1761315300.042952:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315300.042991:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315300.043754:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541632 00000100:00100000:3.0:1761315300.043907:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1575 00000100:00100000:3.0:1761315300.043914:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541632:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315300.043983:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541632:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 69us (1035us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315300.043990:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1575 00000100:00100000:2.0:1761315300.044489:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541760 00000100:00100000:2.0:1761315300.044525:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4219 00000100:00100000:2.0:1761315300.044530:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307541696:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315300.044564:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307541696:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 34us (3598us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315300.044571:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4219 00000100:00100000:2.0:1761315300.044584:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4220 00000100:00100000:2.0:1761315300.044587:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307541760:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315300.044608:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307541760:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 20us (1618us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315300.044614:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4220 00000100:00100000:1.0:1761315300.046302:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315300.046868:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541824 00000100:00100000:0.0:1761315300.047410:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2071 00000100:00100000:0.0:1761315300.047416:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307541824:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315300.047502:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315300.047574:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307541824:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 156us (1278us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315300.047581:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2071 00010000:00010000:0.0:1761315300.059503:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315300.573124:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df11d5eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322507456:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315300.573188:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315300.573288:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507456 00000100:00100000:0.0:1761315300.573337:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2647 00000100:00100000:0.0:1761315300.573379:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322507456:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315300.573410:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315300.573464:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322507456:12345-0@lo:41:osp-pre-1-0.0 Request processed in 121us (277us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315300.573471:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2647 00000100:00100000:3.0:1761315300.573652:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df11d5eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322507456:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315301.083531:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315302.109055:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315303.131517:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315303.131607:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322507584:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315303.131608:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315303.131626:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322507712:0@lo:400:kworker/u8:1.0 00000100:00080000:2.0:1761315303.131634:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315303.131639:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315303.131643:0:36:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315303.131649:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3545 00000100:00080000:2.0:1761315303.131652:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315303.131658:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315303.131686:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007dac8938 x1846869322507904/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315303.131700:0:36:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315303.131703:0:36:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315303.131704:0:36:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315303.131707:0:36:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3545 00000100:00080000:2.0:1761315303.131709:0:36:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315303.131712:0:36:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315303.131723:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315303.131724:0:36:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000fafad97 x1846869322507968/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315303.131756:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024d2fcef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322507520:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315303.131773:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004012701c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322507648:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.131788:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315303.131813:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315303.131815:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002bc728f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322508096:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.131821:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315303.131838:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9dc697d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322508032:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.131853:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315303.131856:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000480f4338 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322507840:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315303.131880:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dac8938 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322507904:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315303.131900:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315303.131909:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aedb9ccf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322507776:0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.131914:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315303.131919:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315303.131935:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f98a942 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322508224:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315303.131944:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fafad97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322507968:0@lo:8:kworker/u8:1.0 00000100:00100000:3.0:1761315303.131950:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315303.131967:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315303.131977:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd20acc0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322508160:0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315303.131996:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315303.132022:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315303.132044:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507648 00000100:00100000:3.0:1761315303.132063:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508096 00000100:00100000:2.0:1761315303.132083:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507776 00000100:00100000:2.0:1761315303.132100:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4221 00000100:00100000:2.0:1761315303.132105:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a244fd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322507648:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:2.0:1761315303.132147:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a244fd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322507648:12345-0@lo:400:kworker/u8:1.0 Request processed in 40us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315303.132154:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4221 00000100:00100000:2.0:1761315303.132168:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4222 00000100:00100000:2.0:1761315303.132172:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322507776:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.132179:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507712 00000100:00100000:3.0:1761315303.132207:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4223 00000100:00100000:2.0:1761315303.132211:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322507776:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (264us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315303.132217:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322508096:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.132218:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507584 00000100:00100000:2.0:1761315303.132223:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4222 00000100:00100000:0.0:1761315303.132234:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507840 00000100:00100000:0.0:1761315303.132258:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2648 00000100:00100000:0.0:1761315303.132262:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322507712:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.132266:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322508096:12345-0@lo:400:kworker/u8:1.0 Request processed in 51us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315303.132275:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4223 00000100:00100000:0.0:1761315303.132300:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322507712:12345-0@lo:400:kworker/u8:1.0 Request processed in 36us (579us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315303.132307:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2649 00000100:00100000:3.0:1761315303.132312:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322507584:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315303.132336:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315303.132398:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322507584:12345-0@lo:41:osp-pre-0-1.0 Request processed in 85us (582us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315303.132738:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2648 00000100:00100000:0.0:1761315303.132755:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2650 00000100:00100000:0.0:1761315303.132759:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e57767e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322507840:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.132787:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e57767e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322507840:12345-0@lo:400:kworker/u8:1.0 Request processed in 28us (877us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315303.132793:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2650 00000100:00100000:0.0:1761315303.132811:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508224 00000100:00100000:0.0:1761315303.132835:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507968 00000100:00100000:0.0:1761315303.132849:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508160 00000100:00100000:3.0:1761315303.132859:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2649 00000100:00100000:0.0:1761315303.132863:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1409 00000100:00100000:0.0:1761315303.132868:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322508224:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.132888:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507520 00000100:00100000:1.0:1761315303.132898:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1410 00000100:00100000:0.0:1761315303.132903:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322508224:12345-0@lo:400:kworker/u8:1.0 Request processed in 33us (936us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315303.132907:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322508160:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.132915:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1409 00000100:00100000:3.0:1761315303.132919:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1576 00000100:00100000:3.0:1761315303.132924:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322507520:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:0.0:1761315303.132933:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2072 00000100:00100000:0.0:1761315303.132938:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322507968:12345-0@lo:8:kworker/u8:1.0 00000100:00100000:1.0:1761315303.132959:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322508160:12345-0@lo:400:kworker/u8:1.0 Request processed in 53us (938us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315303.132968:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322507520:12345-0@lo:400:kworker/u8:1.0 Request processed in 43us (1180us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315303.132972:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1410 00000100:00100000:3.0:1761315303.132981:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1576 00000100:00100000:3.0:1761315303.133030:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315303.133063:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000480f4338 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322507840:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.133067:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004070f96b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322508288:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315303.133078:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f98a942 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322508224:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133092:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322507584:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315303.133113:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315303.133125:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508032 00000100:00100000:3.0:1761315303.133144:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004012701c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322507648:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133155:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4224 00000100:00100000:1.0:1761315303.133160:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322508032:12345-0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.133189:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aedb9ccf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322507776:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133202:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322508032:12345-0@lo:400:kworker/u8:1.0 Request processed in 39us (170us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315303.133207:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd20acc0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322508160:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133212:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4224 00000100:00100000:3.0:1761315303.133222:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002bc728f5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322508096:0@lo:400:kworker/u8:1.0 00000100:00100000:3.0:1761315303.133235:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322507712:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133239:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508288 00000100:00100000:3.0:1761315303.133245:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024d2fcef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322507520:0@lo:400:kworker/u8:1.0 00000100:00100000:1.0:1761315303.133271:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 938 00000100:00100000:1.0:1761315303.133277:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322508288:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315303.133293:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9dc697d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322508032:0@lo:400:kworker/u8:1.0 00002000:00000024:1.0:1761315303.133297:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315303.133340:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322508288:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315303.133374:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 938 00000100:00100000:3.0:1761315303.133407:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004070f96b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322508288:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315303.134229:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322507904 00000100:00100000:2.0:1761315303.134273:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2073 00000100:00100000:2.0:1761315303.134277:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322507904:12345-0@lo:8:kworker/u8:1.0 00010000:02020000:2.0:1761315303.134379:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315303.134427:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322507904:12345-0@lo:8:kworker/u8:1.0 Request processed in 148us (2508us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315303.134433:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2073 00000100:00080000:2.0:1761315303.134470:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315303.134474:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315303.134478:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dac8938 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322507904:0@lo:8:kworker/u8:1.0 00010000:02020000:0.0:1761315303.134840:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315303.134889:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322507968:12345-0@lo:8:kworker/u8:1.0 Request processed in 1951us (2905us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315303.134897:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2072 00000100:00080000:2.0:1761315303.135222:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315303.135225:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315303.135230:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fafad97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322507968:0@lo:8:kworker/u8:1.0 00010000:00010000:0.0:1761315304.156071:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315305.174591:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315305.174966:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315305.175193:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542080 00000100:00100000:1.0:1761315305.175250:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2074 00000100:00100000:1.0:1761315305.175260:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307542080:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315305.175401:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315305.175492:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307542080:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 230us (907us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315305.175503:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2074 00000100:00100000:3.0:1761315305.176025:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315305.176063:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315305.176308:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541888 00000100:00100000:1.0:1761315305.176374:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1577 00000100:00100000:1.0:1761315305.176386:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315305.176448:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307541888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (424us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315305.176460:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1577 00000100:00100000:0.0:1761315305.176673:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307541952 00000100:00100000:2.0:1761315305.176678:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542016 00000100:00100000:0.0:1761315305.176705:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4225 00000100:00100000:0.0:1761315305.176710:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002b7a700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307541952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315305.176722:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4226 00000100:00100000:2.0:1761315305.176729:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d164467 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307542016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315305.176752:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002b7a700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307541952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 40us (1786us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315305.176760:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4225 00000100:00100000:2.0:1761315305.176767:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d164467 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307542016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (702us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315305.176775:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4226 00010000:00010000:2.0:1761315305.179674:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315305.692951:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315305.692982:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322508352:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315305.693050:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315305.693432:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508352 00000100:00100000:2.0:1761315305.693487:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2651 00000100:00100000:2.0:1761315305.693495:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322508352:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315305.693528:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315305.693587:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322508352:12345-0@lo:41:osp-pre-1-0.0 Request processed in 90us (536us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315305.693594:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2651 00000100:00100000:1.0:1761315305.693626:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322508352:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315306.203499:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315307.227568:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315308.251789:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc01d0b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322508416:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315308.251817:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315308.251823:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315308.251827:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315308.251832:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3550 00000100:00080000:0.0:1761315308.251835:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315308.251841:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315308.251862:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000061a066c1 x1846869322508736/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315308.251879:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315308.251882:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315308.251884:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315308.251886:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3550 00000100:00080000:0.0:1761315308.251888:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315308.251891:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315308.251903:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a8a536d5 x1846869322508800/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315308.251939:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315308.251990:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322508672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.252017:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315308.252024:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508416 00000100:00100000:1.0:1761315308.252035:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322509056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.252060:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315308.252063:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1578 00000100:00100000:0.0:1761315308.252070:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322508416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.252080:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315308.252087:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322508608:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.252121:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315308.252126:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322508416:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (189us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315308.252135:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1578 00000100:00100000:1.0:1761315308.252139:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322508992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.252165:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315308.252180:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509056 00000100:00100000:1.0:1761315308.252196:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc01d0b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322508416:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315308.252204:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508992 00000100:00100000:0.0:1761315308.252226:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1411 00000100:00100000:0.0:1761315308.252230:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322509056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315308.252269:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322509056:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (209us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.252279:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1412 00000100:00100000:1.0:1761315308.252285:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322508992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315308.252292:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1411 00000100:00100000:0.0:1761315308.252335:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322508992:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315308.252586:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000349be38c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322508480:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315308.252800:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322508544:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.255458:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322508992:12345-0@lo:400:kworker/u8:2.0 Request processed in 3164us (3285us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.255472:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1412 00000100:00100000:2.0:1761315308.255520:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1761315308.255551:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315308.255558:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322508864:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.255566:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322508736:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315308.255597:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315308.255612:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315308.255644:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508736 00000100:00100000:1.0:1761315308.255651:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322508800:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315308.255679:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2075 00000100:00100000:1.0:1761315308.255681:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315308.255688:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322508736:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315308.255722:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508800 00000100:00100000:1.0:1761315308.255743:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2076 00000100:00100000:1.0:1761315308.255747:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322508800:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315308.255891:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315308.255917:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315308.255932:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322508736:12345-0@lo:8:kworker/u8:2.0 Request processed in 244us (322us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315308.255940:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2075 00000100:00100000:1.0:1761315308.255981:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322508800:12345-0@lo:8:kworker/u8:2.0 Request processed in 202us (272us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315308.255988:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2076 00000100:00080000:2.0:1761315308.255996:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315308.256001:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315308.256006:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322508736:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315308.256019:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1761315308.256020:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508608 00000100:00080000:2.0:1761315308.256023:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315308.256027:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322508800:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315308.256054:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508480 00000100:00100000:2.0:1761315308.256059:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508864 00000100:00100000:1.0:1761315308.256080:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4227 00000100:00100000:1.0:1761315308.256086:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dac8938 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322508608:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315308.256092:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4228 00000100:00100000:2.0:1761315308.256098:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000509695a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322508480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.256135:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dac8938 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322508608:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (4015us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.256142:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4227 00000100:00100000:2.0:1761315308.256148:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000509695a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322508480:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (629us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315308.256156:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4228 00000100:00100000:1.0:1761315308.256163:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4229 00000100:00100000:1.0:1761315308.256168:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064ec7a9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322508864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315308.256193:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000349be38c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322508480:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.256202:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064ec7a9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322508864:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315308.256208:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322508864:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.256211:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4229 00000100:00100000:1.0:1761315308.256248:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508672 00000100:00100000:1.0:1761315308.256276:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2652 00000100:00100000:1.0:1761315308.256282:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004070f96b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322508672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315308.256308:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322509120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315308.256315:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004070f96b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322508672:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (4298us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.256321:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2652 00000100:00100000:2.0:1761315308.256341:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315308.256387:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315308.256394:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322509184:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315308.256410:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509120 00000100:00100000:2.0:1761315308.256420:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315308.256448:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 939 00000100:00100000:1.0:1761315308.256453:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a06c7ae2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322509120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315308.256475:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315308.256522:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a06c7ae2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322509120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (182us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.256528:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 939 00000100:00100000:1.0:1761315308.256547:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509184 00000100:00100000:1.0:1761315308.256566:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2653 00000100:00100000:1.0:1761315308.256569:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007349fc1f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322509184:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315308.256586:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315308.256612:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007349fc1f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322509184:12345-0@lo:41:osp-pre-0-1.0 Request processed in 42us (193us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.256618:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2653 00000100:00100000:0.0:1761315308.256742:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322509056:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315308.256768:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a70ec9ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322508608:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315308.256779:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322508672:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315308.257168:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315308.257194:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322508928:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315308.257216:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315308.257239:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508928 00000100:00100000:3.0:1761315308.257259:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4230 00000100:00100000:3.0:1761315308.257263:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f48f386 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322508928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315308.257287:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f48f386 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322508928:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (72us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315308.257293:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4230 00000100:00100000:3.0:1761315308.257314:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322508928:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.257579:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322508544 00000100:00100000:1.0:1761315308.257610:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2654 00000100:00100000:1.0:1761315308.257614:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322508544:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315308.258111:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322508544:12345-0@lo:400:kworker/u8:2.0 Request processed in 495us (943us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315308.258119:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2654 00000100:00100000:3.0:1761315308.258155:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322508544:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315308.259279:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d148ece5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322509184:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315308.259295:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322509120:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315309.276274:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315310.281769:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315310.281822:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315310.281843:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542336 00000100:00100000:3.0:1761315310.281860:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315310.281891:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2077 00000100:00100000:1.0:1761315310.281900:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307542336:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00000100:00100000:3.0:1761315310.282000:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00010000:02020000:1.0:1761315310.282028:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315310.282029:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542272 00000100:00100000:3.0:1761315310.282065:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542208 00000100:00100000:3.0:1761315310.282086:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4231 00000100:00100000:3.0:1761315310.282091:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009edc9989 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307542272:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315310.282098:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307542336:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 194us (334us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315310.282107:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2077 00000100:00100000:1.0:1761315310.282134:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4232 00000100:00100000:3.0:1761315310.282139:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009edc9989 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307542272:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 47us (279us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315310.282142:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542208:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315310.282152:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4231 00000100:00100000:1.0:1761315310.282181:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542208:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 40us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315310.282188:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4232 00000100:00100000:2.0:1761315310.282333:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542144 00000100:00100000:2.0:1761315310.282401:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1579 00000100:00100000:2.0:1761315310.282407:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307542144:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315310.282455:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307542144:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 48us (634us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315310.282462:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1579 00010000:00010000:0.0:1761315310.299497:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315310.811609:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322509248:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315310.811676:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315310.811796:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509248 00000100:00100000:2.0:1761315310.811846:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2655 00000100:00100000:2.0:1761315310.811852:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322509248:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315310.811882:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315310.811960:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322509248:12345-0@lo:41:osp-pre-1-0.0 Request processed in 79us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315310.811967:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2655 00000100:00100000:3.0:1761315310.812050:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322509248:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315311.323605:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315312.348079:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315313.371511:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315313.371612:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509312:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315313.371687:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315313.372184:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322509376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372219:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509568:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315313.372227:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315313.372249:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315313.372256:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322509440:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372262:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315313.372296:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315313.372298:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315313.372306:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509504:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372337:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315313.372402:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509312 00000100:00100000:1.0:1761315313.372447:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 940 00000100:00100000:1.0:1761315313.372454:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aedb9ccf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322509312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315313.372483:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315313.372527:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aedb9ccf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322509312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (840us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315313.372537:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 940 00000100:00100000:2.0:1761315313.372544:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509376 00000100:00100000:1.0:1761315313.372567:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509568 00000100:00100000:2.0:1761315313.372574:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1580 00000100:00100000:2.0:1761315313.372579:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322509376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372599:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509504 00000100:00100000:2.0:1761315313.372615:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322509376:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (391us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315313.372622:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1580 00000100:00100000:1.0:1761315313.372626:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2656 00000100:00100000:1.0:1761315313.372632:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322509568:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315313.372653:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509440 00000004:00000024:1.0:1761315313.372656:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315313.372682:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4233 00000100:00100000:2.0:1761315313.372687:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d148ece5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322509440:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372692:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322509568:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (433us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315313.372700:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2656 00000100:00100000:1.0:1761315313.372716:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2657 00000100:00100000:1.0:1761315313.372721:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322509504:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315313.372726:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d148ece5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322509440:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (427us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315313.372734:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4233 00000100:00100000:1.0:1761315313.372758:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322509504:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (419us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315313.372765:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2657 00000100:00100000:1.0:1761315313.372809:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509504:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315313.372822:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322509440:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372832:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509568:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315313.372839:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322509376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.372854:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322509312:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315313.373609:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315313.374317:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322509696:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.374397:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315313.374421:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322509632:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.374440:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315313.374538:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509696 00000100:00100000:2.0:1761315313.374575:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2658 00000100:00100000:2.0:1761315313.374580:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322509696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315313.374619:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322509696:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (223us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315313.374625:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2658 00000100:00080000:0.0:1761315313.375022:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315313.375027:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315313.375030:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315313.375035:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3555 00000100:00080000:0.0:1761315313.375038:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315313.375042:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315313.375063:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006e2133c1 x1846869322509760/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315313.375079:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315313.375082:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315313.375083:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315313.375085:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3555 00000100:00080000:0.0:1761315313.375087:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315313.375089:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315313.375097:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000408c4763 x1846869322509824/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315313.375149:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322509760:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315313.375180:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315313.375196:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322509824:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315313.375215:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315313.375245:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509760 00000100:00100000:1.0:1761315313.375468:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322509696:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.375482:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315313.375487:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322510016:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.375512:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315313.375682:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322509888:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315313.375708:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315313.377086:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509824 00000100:00100000:0.0:1761315313.377225:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510016 00000100:00100000:3.0:1761315313.377253:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509632 00000100:00100000:0.0:1761315313.377265:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1413 00000100:00100000:0.0:1761315313.377271:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000349be38c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322510016:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315313.377282:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509888 00000100:00100000:3.0:1761315313.377298:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4234 00000100:00100000:3.0:1761315313.377301:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a70ec9ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322509632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.377520:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2078 00000100:00100000:1.0:1761315313.377524:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322509760:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315313.377617:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315313.377764:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a70ec9ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322509632:12345-0@lo:400:kworker/u8:2.0 Request processed in 460us (3322us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315313.377771:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4234 00000100:00100000:3.0:1761315313.377787:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4235 00000100:00100000:3.0:1761315313.377791:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041264762 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322509888:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315313.377817:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041264762 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322509888:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (2108us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315313.377822:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4235 00000100:00100000:3.0:1761315313.378030:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2079 00000100:00100000:3.0:1761315313.378033:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322509824:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315313.378104:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315313.378270:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322509760:12345-0@lo:8:kworker/u8:2.0 Request processed in 744us (3090us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315313.378271:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322509888:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.378277:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2078 00000100:00100000:2.0:1761315313.378295:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315313.378302:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322509952:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315313.378305:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322509632:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315313.378333:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315313.378365:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761315313.378546:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000349be38c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322510016:12345-0@lo:400:kworker/u8:2.0 Request processed in 1274us (3033us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315313.378554:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1413 00000100:00100000:0.0:1761315313.378589:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322510016:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315313.378626:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322509952 00000100:00100000:0.0:1761315313.378653:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4236 00000100:00100000:0.0:1761315313.378657:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095e7dbc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322509952:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315313.378692:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095e7dbc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322509952:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315313.378698:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4236 00000100:00100000:2.0:1761315313.378702:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322509952:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315313.378741:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315313.378747:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322509760:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315313.378868:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322509824:12345-0@lo:8:kworker/u8:2.0 Request processed in 833us (3651us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315313.378875:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2079 00000100:00080000:1.0:1761315313.378950:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315313.378953:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315313.378957:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322509824:0@lo:8:kworker/u8:2.0 00010000:00010000:1.0:1761315314.396445:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315315.403851:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315315.404018:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542400 00000100:00100000:2.0:1761315315.404077:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1581 00000100:00100000:2.0:1761315315.404083:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307542400:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315315.405734:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307542400:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1644us (1883us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315315.405749:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1581 00000100:00100000:3.0:1761315315.406214:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315315.406331:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315315.406365:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315315.406390:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542464 00000100:00100000:3.0:1761315315.406424:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542528 00000100:00100000:3.0:1761315315.406440:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4237 00000100:00100000:3.0:1761315315.406445:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307542464:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315315.406476:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4238 00000100:00100000:1.0:1761315315.406484:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307542528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315315.406499:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307542464:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 52us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315315.406505:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4237 00000100:00100000:1.0:1761315315.406523:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307542528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 38us (156us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315315.406530:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4238 00000100:00100000:0.0:1761315315.407434:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542592 00000100:00100000:0.0:1761315315.407483:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2080 00000100:00100000:0.0:1761315315.407488:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307542592:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315315.407590:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315315.407649:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307542592:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 160us (1318us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315315.407659:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2080 00010000:00010000:0.0:1761315315.419686:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315315.931597:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322510080:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315315.931662:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315315.931761:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510080 00000100:00100000:3.0:1761315315.931807:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2659 00000100:00100000:3.0:1761315315.931812:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322510080:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315315.931840:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315315.931882:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322510080:12345-0@lo:41:osp-pre-1-0.0 Request processed in 69us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315315.931888:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2659 00000100:00100000:1.0:1761315315.932010:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322510080:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315316.444291:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315317.467877:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315318.492173:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315318.492453:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315318.492469:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322510208:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.492543:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315318.492663:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322510144:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315318.492699:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315318.493401:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:0.0:1761315318.493407:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315318.493413:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761315318.493414:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322510336:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315318.493418:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315318.493425:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3560 00000100:00080000:0.0:1761315318.493428:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315318.493433:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315318.493453:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315318.493455:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000024a427eb x1846869322510464/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315318.493472:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315318.493473:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322510272:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315318.493478:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315318.493481:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315318.493484:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3560 00000100:00080000:0.0:1761315318.493486:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315318.493490:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315318.493503:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005ee7850a x1846869322510528/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315318.493504:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315318.493534:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315318.493544:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322510720:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493554:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510144 00000100:00100000:1.0:1761315318.493573:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315318.493596:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322510656:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315318.493597:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315318.493602:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2660 00000100:00100000:3.0:1761315318.493608:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322510144:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315318.493624:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:3.0:1761315318.493639:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315318.493655:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322510464:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493679:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322510144:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (978us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315318.493685:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322510400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493688:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2660 00000100:00100000:1.0:1761315318.493694:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315318.493717:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322510528:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493720:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510336 00000100:00100000:1.0:1761315318.493755:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315318.493760:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510272 00000100:00100000:0.0:1761315318.493772:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315318.493784:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510656 00000100:00100000:1.0:1761315318.493796:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4239 00000100:00100000:1.0:1761315318.493803:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322510336:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493806:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4240 00000100:00100000:3.0:1761315318.493813:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322510272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.493815:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004eb8e25e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322510784:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315318.493859:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322510336:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315318.493864:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322510272:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (360us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315318.493873:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4239 00000100:00100000:0.0:1761315318.493894:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315318.493894:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4240 00000100:00100000:3.0:1761315318.493910:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4241 00000100:00100000:3.0:1761315318.493913:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322510656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.493938:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315318.493944:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322510656:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315318.493946:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322510592:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.493953:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4241 00000100:00100000:3.0:1761315318.493974:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510400 00000100:00100000:0.0:1761315318.493978:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315318.493997:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2661 00000100:00100000:3.0:1761315318.494002:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322510400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494036:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322510400:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (289us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315318.494051:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2661 00000100:00100000:0.0:1761315318.494060:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510720 00000100:00100000:3.0:1761315318.494067:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510592 00000100:00100000:3.0:1761315318.494093:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4242 00000100:00100000:3.0:1761315318.494096:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322510592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.494108:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510464 00000100:00100000:3.0:1761315318.494121:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322510592:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (146us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315318.494129:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4242 00000100:00100000:0.0:1761315318.494148:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510528 00000100:00100000:3.0:1761315318.494165:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322510592:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494177:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322510400:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.494180:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510784 00000100:00100000:0.0:1761315318.494203:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1414 00000100:00100000:0.0:1761315318.494208:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322510720:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494225:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1415 00000100:00100000:3.0:1761315318.494229:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322510784:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.494248:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322510720:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (674us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315318.494255:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1414 00000100:00100000:3.0:1761315318.494269:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322510784:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315318.494274:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1415 00000100:00100000:0.0:1761315318.494290:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2081 00000100:00100000:3.0:1761315318.494297:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2082 00000100:00100000:0.0:1761315318.494298:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322510464:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494305:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322510528:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315318.494441:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315318.494468:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322510528:12345-0@lo:8:kworker/u8:2.0 Request processed in 164us (714us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315318.494473:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2082 00000100:00100000:3.0:1761315318.494500:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322510720:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494511:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322510656:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315318.494525:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1761315318.494527:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315318.494528:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322510336:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315318.494535:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315318.494542:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322510528:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315318.494542:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322510272:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315318.494572:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315318.494577:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315318.494582:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322510464:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315318.494586:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322510464:12345-0@lo:8:kworker/u8:2.0 Request processed in 287us (892us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315318.494590:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004eb8e25e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322510784:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315318.494600:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2081 00000100:00100000:1.0:1761315318.494633:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322510144:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315318.496431:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510208 00000100:00100000:2.0:1761315318.496468:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1582 00000100:00100000:2.0:1761315318.496474:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322510208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315318.496535:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322510208:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (3993us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315318.496541:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1582 00000100:00100000:1.0:1761315318.496801:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322510848:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315318.496837:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315318.499425:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510848 00000100:00100000:2.0:1761315318.499491:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 941 00000100:00100000:2.0:1761315318.499497:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322510848:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315318.499506:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322510208:0@lo:400:kworker/u8:2.0 00002000:00000024:2.0:1761315318.499528:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315318.499602:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322510848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (2764us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315318.499610:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 941 00000100:00100000:3.0:1761315318.499645:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc30b944 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322510848:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315319.515564:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315320.519683:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315320.520275:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542656 00000100:00100000:1.0:1761315320.520382:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1583 00000100:00100000:1.0:1761315320.520390:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307542656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315320.520476:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307542656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 86us (800us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315320.520484:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1583 00000100:00100000:3.0:1761315320.522489:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315320.522667:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315320.523824:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315320.525474:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542848 00000100:00100000:0.0:1761315320.525539:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2083 00000100:00100000:0.0:1761315320.525548:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307542848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315320.525654:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315320.525760:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307542848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 212us (1944us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315320.525769:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2083 00000100:00100000:2.0:1761315320.528095:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542720 00000100:00100000:2.0:1761315320.528165:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542784 00000100:00100000:2.0:1761315320.528200:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4243 00000100:00100000:2.0:1761315320.528206:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7db22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315320.528295:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7db22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (5812us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315320.528303:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4243 00000100:00100000:2.0:1761315320.528324:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4244 00000100:00100000:2.0:1761315320.528327:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d26c929f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307542784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315320.528411:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d26c929f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307542784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 82us (5745us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315320.528418:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4244 00010000:00010000:1.0:1761315320.541532:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315321.051641:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004eb8e25e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322510912:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315321.051734:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315321.051857:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510912 00000100:00100000:3.0:1761315321.051956:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2662 00000100:00100000:3.0:1761315321.051965:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc30b944 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322510912:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315321.052013:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315321.056709:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004eb8e25e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322510912:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315321.056803:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc30b944 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322510912:12345-0@lo:41:osp-pre-1-0.0 Request processed in 4836us (5068us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315321.056816:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2662 00010000:00010000:3.0:1761315321.563529:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315322.587665:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315323.612973:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315323.613048:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315323.613076:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322510976:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315323.613166:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315323.613281:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322510976 00000100:00100000:1.0:1761315323.613360:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2663 00000100:00100000:1.0:1761315323.613367:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322510976:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315323.613409:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315323.613458:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bc728f5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322510976:12345-0@lo:41:osp-pre-0-1.0 Request processed in 90us (294us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315323.613466:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2663 00000100:00100000:2.0:1761315323.613562:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322510976:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315323.614285:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315323.614291:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315323.614296:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315323.614303:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3566 00000100:00080000:3.0:1761315323.614306:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315323.614311:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315323.614333:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322511168:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.614334:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009e135009 x1846869322511360/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315323.614364:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315323.614367:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315323.614369:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315323.614372:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3566 00000100:00080000:3.0:1761315323.614375:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315323.614378:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315323.614390:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000aa10021 x1846869322511424/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315323.614397:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315323.614421:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322511552:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315323.614442:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315323.614555:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322511104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315323.614591:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322511360:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315323.614614:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315323.614630:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322511424:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315323.614645:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.614899:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322511040:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.614940:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.614957:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322511296:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315323.614986:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.614998:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322511680:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.615017:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.615065:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511360 00000100:00100000:0.0:1761315323.615109:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511424 00000100:00100000:2.0:1761315323.615110:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315323.615117:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322511232:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.615127:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511680 00000100:00100000:2.0:1761315323.615143:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.615153:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1416 00000100:00100000:2.0:1761315323.615157:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322511616:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.615159:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322511680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315323.615178:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.615204:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322511680:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315323.615211:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1416 00000100:00100000:0.0:1761315323.615225:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511616 00000100:00100000:0.0:1761315323.615237:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1417 00000100:00100000:0.0:1761315323.615241:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322511616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.615292:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511168 00000100:00100000:1.0:1761315323.616233:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511552 00000100:00100000:3.0:1761315323.616236:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4245 00000100:00100000:3.0:1761315323.616242:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322511168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.616298:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd20acc0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322511168:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (1901us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315323.616302:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322511616:12345-0@lo:400:kworker/u8:0.0 Request processed in 1060us (1123us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315323.616305:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4245 00000100:00100000:0.0:1761315323.616308:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1417 00000100:00100000:3.0:1761315323.616322:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4246 00000100:00100000:0.0:1761315323.616323:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2084 00000100:00100000:0.0:1761315323.616326:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322511360:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315323.616326:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f98a942 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322511552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.616365:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f98a942 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322511552:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (1923us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315323.616371:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4246 00000100:00100000:2.0:1761315323.616415:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511232 00010000:02020000:0.0:1761315323.616437:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315323.616444:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2664 00000100:00100000:2.0:1761315323.616448:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322511232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.616475:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322511360:12345-0@lo:8:kworker/u8:0.0 Request processed in 148us (1861us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315323.616475:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322511232:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (1333us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315323.616481:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2664 00000100:00100000:0.0:1761315323.616485:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2084 00000100:00100000:0.0:1761315323.616494:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2085 00000100:00100000:0.0:1761315323.616498:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322511424:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315323.616577:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315323.616604:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322511424:12345-0@lo:8:kworker/u8:0.0 Request processed in 105us (1958us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315323.616609:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2085 00000100:00100000:0.0:1761315323.616630:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511040 00000100:00100000:0.0:1761315323.616657:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1584 00000100:00100000:0.0:1761315323.616661:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322511040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.616695:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322511040:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (1756us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315323.616701:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1584 00000100:00100000:0.0:1761315323.616735:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511296 00000100:00100000:0.0:1761315323.616758:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 942 00000100:00100000:0.0:1761315323.616762:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322511296:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315323.616783:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315323.616815:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322511296:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (1830us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315323.616821:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 942 00000100:00080000:0.0:1761315323.616895:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315323.616900:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315323.616905:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322511360:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315323.616916:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315323.616919:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315323.616922:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322511424:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315323.616965:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322511040:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.616983:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b30ed0e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322511296:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315323.616994:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322511680:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.617832:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315323.620618:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322511488:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315323.620703:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315323.621935:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511104 00000100:00100000:3.0:1761315323.622019:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4247 00000100:00100000:3.0:1761315323.622027:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322511104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.622125:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322511104:12345-0@lo:400:kworker/u8:0.0 Request processed in 98us (4297us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315323.622133:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4247 00000100:00100000:3.0:1761315323.622278:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322511232:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.622294:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322511616:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.622306:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322511552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315323.622320:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322511168:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315323.623715:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511488 00000100:00100000:0.0:1761315323.623764:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322511104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315323.623782:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4248 00000100:00100000:1.0:1761315323.623788:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322511488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315323.623883:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b30ed0e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322511488:12345-0@lo:400:kworker/u8:0.0 Request processed in 93us (3179us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315323.623889:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4248 00000100:00100000:0.0:1761315323.623915:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322511488:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315324.635582:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315325.640140:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315325.640202:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315325.640239:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315325.640285:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315325.640338:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543104 00000100:00100000:3.0:1761315325.640435:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2086 00000100:00100000:3.0:1761315325.640443:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307543104:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315325.640541:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315325.640615:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307543104:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 172us (330us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315325.640624:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2086 00000100:00100000:1.0:1761315325.640855:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543040 00000100:00100000:1.0:1761315325.640902:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542976 00000100:00100000:1.0:1761315325.640922:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4249 00000100:00100000:1.0:1761315325.640926:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307543040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315325.640977:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307543040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (776us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315325.640986:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4249 00000100:00100000:1.0:1761315325.641000:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4250 00000100:00100000:1.0:1761315325.641004:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315325.641027:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307542976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 22us (789us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315325.641033:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4250 00000100:00100000:0.0:1761315325.642975:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307542912 00000100:00100000:0.0:1761315325.643075:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1585 00000100:00100000:0.0:1761315325.643083:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307542912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315325.643189:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307542912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 106us (3064us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315325.643198:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1585 00010000:00010000:1.0:1761315325.661675:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315326.174407:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315326.174433:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322511744:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315326.174518:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315326.174669:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511744 00000100:00100000:3.0:1761315326.174725:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2665 00000100:00100000:3.0:1761315326.174733:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075457230 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322511744:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315326.174773:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315326.174840:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075457230 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322511744:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (321us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315326.174847:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2665 00000100:00100000:0.0:1761315326.174976:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322511744:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315326.683585:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315327.707690:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315328.731558:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322511808:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315328.731590:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315328.731595:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315328.731599:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315328.731605:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3571 00000100:00080000:0.0:1761315328.731608:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315328.731614:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315328.731639:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009e135009 x1846869322512128/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315328.731640:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315328.731658:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315328.731661:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315328.731663:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315328.731667:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3571 00000100:00080000:0.0:1761315328.731669:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315328.731673:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322512064:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315328.731674:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315328.731692:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000007c7b302 x1846869322512192/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315328.731710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315328.731735:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322512448:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315328.731748:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322512128:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315328.731763:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315328.731786:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315328.731787:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315328.731794:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322511872:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315328.731825:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322512192:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315328.731836:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315328.731856:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322512256:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315328.731862:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315328.731889:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315328.731913:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512448 00000100:00100000:3.0:1761315328.731925:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512064 00000100:00100000:0.0:1761315328.731958:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512128 00000100:00100000:3.0:1761315328.731962:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2666 00000100:00100000:3.0:1761315328.731968:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322512064:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315328.731981:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512192 00000100:00100000:0.0:1761315328.731999:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1418 00000100:00100000:0.0:1761315328.732004:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322512448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315328.732015:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322512064:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (306us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315328.732023:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2666 00000100:00100000:0.0:1761315328.732043:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322512448:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (280us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315328.732050:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2087 00000100:00100000:3.0:1761315328.732054:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322512128:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315328.732150:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315328.732179:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322512000:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315328.732185:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322512128:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (400us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315328.732212:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732231:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512384:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.732248:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732257:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315328.732261:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322511936:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.732287:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732315:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322512320:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.732336:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732387:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511808 00000100:00100000:1.0:1761315328.732413:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1586 00000100:00100000:1.0:1761315328.732417:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322511808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.732444:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322511808:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (809us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.732449:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1586 00000100:00100000:1.0:1761315328.732480:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511872 00000100:00100000:1.0:1761315328.732508:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512256 00000100:00100000:1.0:1761315328.732525:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512000 00000100:00100000:1.0:1761315328.732540:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512320 00000100:00100000:1.0:1761315328.732555:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4251 00000100:00100000:1.0:1761315328.732559:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f4362ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322511872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.732576:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315328.732584:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512576:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315328.732593:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f4362ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322511872:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (757us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.732599:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4251 00000100:00100000:2.0:1761315328.732613:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732616:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4252 00000100:00100000:1.0:1761315328.732622:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322512256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.732649:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322511936 00000100:00100000:1.0:1761315328.732655:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322512256:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (766us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.732661:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4252 00000100:00100000:2.0:1761315328.732677:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512576 00000100:00100000:1.0:1761315328.732681:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4253 00000100:00100000:1.0:1761315328.732687:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322512000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.732693:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2667 00000100:00100000:2.0:1761315328.732697:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322511936:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.732726:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322512000:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (513us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315328.732733:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322511936:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (444us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.732736:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4253 00000100:00100000:2.0:1761315328.732744:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2667 00000100:00100000:1.0:1761315328.732761:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4254 00000100:00100000:2.0:1761315328.732763:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2668 00000100:00100000:1.0:1761315328.732773:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322512320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.732773:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322512576:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315328.732804:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315328.732823:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322512320:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (485us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.732830:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4254 00000100:00100000:2.0:1761315328.732848:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322512576:12345-0@lo:41:osp-pre-0-1.0 Request processed in 77us (234us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315328.732855:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2668 00000100:00100000:1.0:1761315328.732868:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000357267b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322512512:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315328.732897:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315328.732929:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512512 00000100:00100000:1.0:1761315328.732952:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 943 00000100:00100000:1.0:1761315328.732955:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322512512:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315328.732973:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315328.733004:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322512512:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (107us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.733009:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 943 00000100:00100000:1.0:1761315328.733045:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000357267b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322512512:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315328.733087:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512384 00000100:00100000:0.0:1761315328.733366:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1418 00000100:00100000:3.0:1761315328.733384:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2087 00000100:00100000:0.0:1761315328.733397:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2088 00000100:00100000:0.0:1761315328.733402:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322512192:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315328.733448:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322511872:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315328.733459:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322512256:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315328.733467:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322512448:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315328.733478:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322512064:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315328.733483:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315328.733488:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000408c4763 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322511808:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315328.733522:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322512192:12345-0@lo:8:kworker/u8:2.0 Request processed in 118us (1663us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315328.733530:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2088 00010000:00010000:0.0:1761315328.733595:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315328.733620:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315328.733624:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315328.733628:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322512128:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315328.733640:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315328.733643:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315328.733646:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322512192:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315328.734765:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512576:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315328.734779:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322511936:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.734788:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322512320:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315328.734797:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322512000:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.738033:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1419 00000100:00100000:1.0:1761315328.738045:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322512384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315328.738156:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322512384:12345-0@lo:400:kworker/u8:2.0 Request processed in 114us (5907us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315328.738164:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1419 00000100:00100000:2.0:1761315328.738276:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512384:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315329.756527:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315330.762229:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315330.762306:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315330.762416:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315330.762418:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543168 00000100:00100000:2.0:1761315330.762477:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543232 00000100:00100000:1.0:1761315330.762496:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1587 00000100:00100000:1.0:1761315330.762505:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307543168:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315330.762592:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307543168:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (287us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315330.762602:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1587 00000100:00100000:0.0:1761315330.763270:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543296 00000100:00100000:0.0:1761315330.763330:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4255 00000100:00100000:0.0:1761315330.763338:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307543232:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315330.763462:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307543232:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 124us (1245us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315330.763470:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4255 00000100:00100000:0.0:1761315330.763490:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4256 00000100:00100000:0.0:1761315330.763494:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307543296:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315330.763520:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307543296:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (1105us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315330.763526:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4256 00000100:00100000:0.0:1761315330.764300:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315330.764384:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543360 00000100:00100000:1.0:1761315330.764426:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2089 00000100:00100000:1.0:1761315330.764431:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307543360:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315330.764541:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315330.764602:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307543360:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 169us (304us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315330.764610:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2089 00010000:00010000:3.0:1761315330.780021:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315331.295586:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315331.297158:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322512640:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315331.297261:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315331.297400:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512640 00000100:00100000:2.0:1761315331.297927:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2669 00000100:00100000:2.0:1761315331.297934:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322512640:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315331.297977:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315331.298125:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322512640:12345-0@lo:41:osp-pre-1-0.0 Request processed in 190us (867us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315331.298133:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2669 00000100:00100000:0.0:1761315331.298204:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322512640:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315331.803984:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315332.827598:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315333.851574:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322512768:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315333.851586:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000357267b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512896:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761315333.851608:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315333.851615:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315333.851619:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315333.851626:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3576 00000100:00080000:1.0:1761315333.851629:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315333.851637:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00010000:00010000:3.0:1761315333.851702:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315333.851715:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322512704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.851718:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000009cdcdea x1846869322513152/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315333.851738:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315333.851741:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315333.851742:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315333.851745:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3576 00000100:00080000:1.0:1761315333.851747:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315333.851751:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315333.851765:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315333.851765:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315333.851767:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fa32dc4d x1846869322513216/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315333.851828:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322513024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.851833:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322512832:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315333.851867:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315333.851879:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322513280:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.851888:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512896 00000100:00100000:0.0:1761315333.851892:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322513408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.851908:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315333.851918:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315333.851935:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315333.851943:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c6196fc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322513344:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.851944:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1761315333.851948:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512768 00000100:00100000:1.0:1761315333.851954:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322512896:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315333.851968:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315333.851980:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2670 00000100:00100000:2.0:1761315333.851986:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322512768:12345-0@lo:41:osp-pre-0-1.0 00002000:00000024:1.0:1761315333.851989:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315333.851991:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322513152:0@lo:8:kworker/u8:2.0 00000004:00000024:2.0:1761315333.852016:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315333.852019:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315333.852046:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322512896:12345-0@lo:13:osp-pre-1-1.0 Request processed in 92us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315333.852051:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa32dc4d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322513216:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852063:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 944 00000100:00100000:2.0:1761315333.852067:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322512768:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (303us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315333.852075:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2670 00000100:00100000:0.0:1761315333.852079:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315333.852099:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513408 00000100:00100000:2.0:1761315333.852105:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513024 00000100:00100000:0.0:1761315333.852110:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513280 00000100:00100000:1.0:1761315333.852131:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513152 00000100:00100000:0.0:1761315333.852143:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513344 00000100:00100000:1.0:1761315333.852158:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513216 00000100:00100000:1.0:1761315333.852176:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1420 00000100:00100000:1.0:1761315333.852180:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322513408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852210:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322513408:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315333.852216:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1420 00000100:00100000:1.0:1761315333.852235:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2090 00000100:00100000:2.0:1761315333.852235:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4257 00000100:00100000:1.0:1761315333.852243:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322513152:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852246:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322513280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852283:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322513280:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (375us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315333.852293:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4257 00000100:00100000:2.0:1761315333.852307:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4258 00000100:00100000:2.0:1761315333.852310:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322513024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852334:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322513024:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (469us total) trans 0 rc 0/0 00010000:02020000:1.0:1761315333.852339:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315333.852368:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4258 00000100:00100000:2.0:1761315333.852383:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4259 00000100:00100000:2.0:1761315333.852387:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322513344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852401:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322513152:12345-0@lo:8:kworker/u8:2.0 Request processed in 158us (381us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315333.852409:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2090 00000100:00100000:1.0:1761315333.852423:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2091 00000100:00100000:2.0:1761315333.852427:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322513344:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (457us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315333.852431:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322513216:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852437:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4259 00000100:00100000:3.0:1761315333.852492:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315333.852493:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322513280:0@lo:400:kworker/u8:2.0 00010000:02020000:1.0:1761315333.852508:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315333.852511:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000357267b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322512896:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315333.852511:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322512960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852547:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322513216:12345-0@lo:8:kworker/u8:2.0 Request processed in 114us (465us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315333.852548:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315333.852551:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c6196fc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322513344:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852560:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2091 00000100:00100000:2.0:1761315333.852567:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001339376 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322513024:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852577:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322513408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852587:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512704 00000100:00100000:2.0:1761315333.852591:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322512768:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315333.852607:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512960 00000100:00100000:1.0:1761315333.852623:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1588 00000100:00100000:1.0:1761315333.852629:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322512704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315333.852637:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2671 00000100:00100000:2.0:1761315333.852642:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315333.852643:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322512960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852662:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322513088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852678:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322512704:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (184us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315333.852684:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322512960:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (138us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315333.852686:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315333.852690:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1588 00000100:00100000:3.0:1761315333.852693:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2671 00000100:00100000:2.0:1761315333.852705:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322513472:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315333.852710:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513088 00000100:00100000:2.0:1761315333.852730:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315333.852735:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2672 00000100:00100000:3.0:1761315333.852739:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322513088:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315333.852748:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315333.852754:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315333.852758:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322513152:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315333.852780:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315333.852783:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315333.852786:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa32dc4d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322513216:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315333.852787:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513472 00000100:00100000:3.0:1761315333.852789:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322513088:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (103us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315333.852798:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2672 00000100:00100000:2.0:1761315333.852812:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1421 00000100:00100000:2.0:1761315333.852816:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322513472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315333.852824:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322512832 00000100:00100000:1.0:1761315333.852826:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322512704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852837:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322512960:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315333.852853:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4260 00000100:00100000:3.0:1761315333.852858:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322512832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315333.852881:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322513088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315333.852895:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322512832:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315333.852903:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4260 00000100:00100000:1.0:1761315333.852906:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322512832:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315333.855114:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322513472:12345-0@lo:400:kworker/u8:2.0 Request processed in 2291us (2379us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315333.855126:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1421 00000100:00100000:1.0:1761315333.855161:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322513472:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315334.875537:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315335.880285:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315335.880375:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315335.880408:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315335.880464:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543488 00000100:00100000:2.0:1761315335.880494:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543552 00000100:00100000:3.0:1761315335.880510:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543424 00000100:00100000:1.0:1761315335.880520:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4261 00000100:00100000:1.0:1761315335.880528:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307543488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315335.880534:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4262 00000100:00100000:2.0:1761315335.880541:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307543552:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315335.880552:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1589 00000100:00100000:3.0:1761315335.880558:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307543424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315335.880597:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307543488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (318us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315335.880599:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307543552:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (190us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315335.880610:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307543424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315335.880611:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4261 00000100:00100000:3.0:1761315335.880621:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1589 00000100:00100000:1.0:1761315335.880718:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315335.880856:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543616 00000100:00100000:3.0:1761315335.880886:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2092 00000100:00100000:3.0:1761315335.880890:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307543616:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315335.880941:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4262 00010000:02020000:3.0:1761315335.881012:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315335.884321:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307543616:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 3421us (3596us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315335.884337:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2092 00010000:00010000:0.0:1761315335.899730:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315336.411626:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322513536:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315336.411693:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315336.411845:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513536 00000100:00100000:3.0:1761315336.411893:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2673 00000100:00100000:3.0:1761315336.411899:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322513536:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315336.411928:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315336.412135:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322513536:12345-0@lo:41:osp-pre-1-0.0 Request processed in 139us (346us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315336.412164:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea7e262a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322513536:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315336.412208:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2673 00010000:00010000:3.0:1761315336.923506:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315337.951833:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315338.971514:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315338.971587:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315338.971593:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315338.971596:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315338.971605:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3581 00000100:00080000:1.0:1761315338.971607:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315338.971613:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315338.971633:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005c899dd9 x1846869322513920/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315338.971649:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315338.971652:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315338.971653:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315338.971656:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3581 00000100:00080000:1.0:1761315338.971657:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315338.971660:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315338.971671:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007fe660ed x1846869322513984/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315338.971710:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322513792:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.971721:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322513920:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315338.971798:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.971799:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.971830:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322514176:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.971834:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007fe660ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322513984:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315338.971870:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315338.971898:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513920 00000100:00100000:0.0:1761315338.971905:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315338.971912:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322513728:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.971935:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514176 00000100:00100000:0.0:1761315338.971939:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.971961:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000883ca932 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322514112:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.971973:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1422 00000100:00100000:1.0:1761315338.971981:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322514176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.971988:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.972022:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2093 00000100:00100000:0.0:1761315338.972027:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322513920:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315338.972041:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322514176:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315338.972053:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1422 00000100:00100000:1.0:1761315338.972095:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513728 00010000:02020000:0.0:1761315338.972115:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315338.972119:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2674 00000100:00100000:1.0:1761315338.972124:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322513728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972147:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322513920:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (354us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315338.972155:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2093 00000100:00100000:1.0:1761315338.972167:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322513728:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315338.972175:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2674 00000100:00100000:0.0:1761315338.972180:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513792 00000100:00100000:1.0:1761315338.972203:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.972211:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514112 00000100:00100000:0.0:1761315338.972234:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4263 00000100:00100000:0.0:1761315338.972238:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322513792:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315338.972242:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315338.972246:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315338.972252:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322513920:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972269:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322513792:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (474us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315338.972275:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4263 00000100:00100000:0.0:1761315338.972290:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4264 00000100:00100000:0.0:1761315338.972294:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322514112:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.972316:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322513728:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972322:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322514112:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (335us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315338.972332:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4264 00000100:00100000:1.0:1761315338.972335:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000883ca932 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322514112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972378:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513984 00000100:00100000:1.0:1761315338.972381:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322513792:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315338.972396:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322514176:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972400:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2094 00000100:00100000:0.0:1761315338.972405:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df19d595 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322513984:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315338.972486:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315338.972509:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322513600:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972529:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df19d595 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322513984:12345-0@lo:8:kworker/u8:0.0 Request processed in 122us (326us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315338.972536:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2094 00000100:00100000:3.0:1761315338.972554:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315338.972577:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322513856:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.972604:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315338.972626:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322514240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.972652:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315338.972677:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315338.972685:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322513664:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972701:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514240 00000100:00100000:3.0:1761315338.972716:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315338.972727:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1423 00000100:00100000:0.0:1761315338.972743:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322514240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.972751:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043048615 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322514048:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315338.972777:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322514240:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (125us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315338.972786:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1423 00000100:00100000:3.0:1761315338.972787:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315338.972813:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322514240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.972847:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513600 00000100:00100000:3.0:1761315338.972873:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1590 00000100:00100000:3.0:1761315338.972877:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322513600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.972910:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322513600:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (359us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315338.972917:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1590 00000100:00100000:3.0:1761315338.972943:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513664 00000100:00100000:3.0:1761315338.972963:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514048 00000100:00100000:3.0:1761315338.972981:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4265 00000100:00100000:3.0:1761315338.972985:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7b15fb6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322513664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973014:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7b15fb6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322513664:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315338.973020:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4265 00000100:00100000:3.0:1761315338.973031:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4266 00000100:00100000:3.0:1761315338.973034:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322514048:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973058:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322514048:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (275us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315338.973063:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4266 00000100:00100000:3.0:1761315338.973083:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322513856 00000100:00100000:3.0:1761315338.973106:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2675 00000100:00100000:3.0:1761315338.973110:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322513856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973137:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322513856:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (534us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315338.973142:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2675 00000100:00100000:3.0:1761315338.973171:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322513664:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973181:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043048615 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322514048:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973188:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322513600:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315338.973196:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322513856:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315338.973705:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315338.973710:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315338.973714:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007fe660ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322513984:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315338.974841:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315338.974855:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322514304:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315338.974906:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315338.975081:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514304 00000100:00100000:2.0:1761315338.975129:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2676 00000100:00100000:2.0:1761315338.975135:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322514304:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315338.975164:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315338.975218:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322514304:12345-0@lo:41:osp-pre-0-1.0 Request processed in 83us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315338.975226:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2676 00000100:00100000:0.0:1761315338.975523:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322514304:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315338.975610:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322514368:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315338.975643:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315338.975932:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514368 00000100:00100000:3.0:1761315338.975972:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 945 00000100:00100000:3.0:1761315338.975976:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322514368:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315338.975995:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315338.976764:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322514368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 785us (1118us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315338.976774:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 945 00000100:00100000:2.0:1761315338.976809:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322514368:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1761315339.995517:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315341.001406:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315341.002236:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315341.002291:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315341.002542:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543872 00000100:00100000:2.0:1761315341.002599:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2095 00000100:00100000:2.0:1761315341.002608:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307543872:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:2.0:1761315341.002713:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315341.002813:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307543872:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 205us (1474us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315341.002833:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2095 00000100:00100000:3.0:1761315341.003200:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543744 00000100:00100000:3.0:1761315341.003249:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4267 00000100:00100000:3.0:1761315341.003255:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307543744:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315341.003314:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307543744:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 60us (1082us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315341.003321:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4267 00000100:00100000:1.0:1761315341.004355:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315341.004544:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543680 00000100:00100000:2.0:1761315341.004597:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543808 00000100:00100000:0.0:1761315341.004612:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1591 00000100:00100000:0.0:1761315341.004620:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307543680:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315341.004636:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4268 00000100:00100000:2.0:1761315341.004641:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307543808:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315341.005072:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307543680:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 450us (2779us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315341.005076:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307543808:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 430us (737us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315341.005087:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4268 00000100:00100000:0.0:1761315341.005090:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1591 00010000:00010000:0.0:1761315341.019644:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315341.532304:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322514432:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315341.532429:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315341.533495:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514432 00000100:00100000:3.0:1761315341.533571:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2677 00000100:00100000:3.0:1761315341.533578:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007fe660ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322514432:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315341.533631:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315341.533710:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007fe660ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322514432:12345-0@lo:41:osp-pre-1-0.0 Request processed in 131us (1284us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315341.533748:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2677 00000100:00100000:3.0:1761315341.533809:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322514432:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315342.044866:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315343.067660:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315344.094334:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043048615 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322514496:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315344.094418:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315344.094499:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514496 00000100:00100000:1.0:1761315344.094544:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 946 00000100:00100000:1.0:1761315344.094549:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322514496:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315344.094572:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315344.094607:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322514496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315344.094613:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 946 00000100:00100000:2.0:1761315344.094686:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043048615 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322514496:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315344.094880:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315344.094885:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315344.094888:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315344.094893:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3586 00000100:00080000:0.0:1761315344.094896:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315344.094899:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315344.094919:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bfdc3142 x1846869322514880/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315344.094931:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315344.094934:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315344.094935:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315344.094938:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3586 00000100:00080000:0.0:1761315344.094939:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315344.094941:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315344.094951:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000024e9f3ae x1846869322514944/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00010000:00010000:0.0:1761315344.095016:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315344.095077:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322514624:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095105:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095128:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322515008:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095147:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095162:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315344.095165:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322514688:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095182:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095194:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322515072:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095208:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095228:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514624 00000100:00100000:2.0:1761315344.095247:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515008 00000100:00100000:2.0:1761315344.095257:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515072 00000100:00100000:2.0:1761315344.095267:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4269 00000100:00100000:2.0:1761315344.095271:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322514624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095302:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322514624:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095307:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4269 00000100:00100000:2.0:1761315344.095319:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4270 00000100:00100000:2.0:1761315344.095323:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322515008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095364:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322515008:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (217us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095370:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4270 00000100:00100000:2.0:1761315344.095379:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4271 00000100:00100000:2.0:1761315344.095382:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322515072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095401:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322515072:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (193us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095406:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4271 00000100:00100000:2.0:1761315344.095420:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514688 00000100:00100000:2.0:1761315344.095437:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2678 00000100:00100000:2.0:1761315344.095440:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6196fc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322514688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095460:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6196fc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322514688:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (277us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095464:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2678 00000100:00100000:2.0:1761315344.095483:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322514688:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095490:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322515072:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095497:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322514624:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095503:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322515008:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095522:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322514752:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095539:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095550:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2a9d0ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322515136:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095566:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095577:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:2.0:1761315344.095582:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322514560:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095603:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095614:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322514816:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095632:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095644:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322515200:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315344.095662:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315344.095681:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514816 00000100:00100000:2.0:1761315344.095711:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515200 00000100:00100000:2.0:1761315344.095722:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2679 00000100:00100000:2.0:1761315344.095725:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322514816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095756:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322514816:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (124us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095763:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2679 00000100:00100000:2.0:1761315344.095773:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2680 00000100:00100000:2.0:1761315344.095776:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322515200:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315344.095796:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315344.095819:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322515200:12345-0@lo:41:osp-pre-0-1.0 Request processed in 42us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095824:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2680 00000100:00100000:2.0:1761315344.095843:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514752 00000100:00100000:2.0:1761315344.095856:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4272 00000100:00100000:2.0:1761315344.095859:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322514752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095882:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322514752:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (343us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.095888:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4272 00000100:00100000:2.0:1761315344.095910:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322514816:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.095919:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322515200:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315344.095926:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a8504f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322514752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315344.096002:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfdc3142 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322514880:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315344.096030:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315344.096055:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322514944:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315344.096076:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315344.096109:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514560 00000100:00100000:1.0:1761315344.096134:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1592 00000100:00100000:1.0:1761315344.096137:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322514560:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315344.096168:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322514560:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (565us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315344.096173:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1592 00000100:00100000:1.0:1761315344.096210:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322514560:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.096271:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515136 00000100:00100000:2.0:1761315344.096301:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514880 00000100:00100000:2.0:1761315344.096314:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322514944 00000100:00100000:2.0:1761315344.096325:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1424 00000100:00100000:2.0:1761315344.096328:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322515136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315344.096379:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322515136:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (813us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315344.096385:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1424 00000100:00100000:2.0:1761315344.096396:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2096 00000100:00100000:2.0:1761315344.096399:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000883ca932 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322514880:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315344.096485:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315344.096513:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000883ca932 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322514880:12345-0@lo:8:kworker/u8:2.0 Request processed in 113us (483us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315344.096520:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2096 00000100:00100000:2.0:1761315344.096530:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2097 00000100:00100000:2.0:1761315344.096533:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322514944:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315344.096607:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315344.096630:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322514944:12345-0@lo:8:kworker/u8:2.0 Request processed in 96us (554us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315344.096635:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2097 00000100:00080000:1.0:1761315344.096819:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315344.096823:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315344.096827:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfdc3142 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322514880:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315344.096834:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315344.096837:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315344.096839:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322514944:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315344.096867:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2a9d0ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322515136:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315345.115771:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315346.123154:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315346.123229:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315346.123257:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544128 00000100:00100000:3.0:1761315346.123271:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315346.123309:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307543936 00000100:00100000:3.0:1761315346.123323:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315346.123335:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2098 00000100:00100000:0.0:1761315346.123377:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307544128:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315346.123383:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1593 00000100:00100000:2.0:1761315346.123387:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307543936:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315346.123463:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307543936:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 74us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315346.123469:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1593 00010000:02020000:0.0:1761315346.123470:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315346.123496:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307544128:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 152us (355us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315346.123502:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2098 00000100:00100000:0.0:1761315346.123527:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544064 00000100:00100000:2.0:1761315346.124511:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544000 00000100:00100000:2.0:1761315346.124541:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4273 00000100:00100000:2.0:1761315346.124546:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307544064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315346.124595:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307544064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 48us (1325us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315346.124601:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4273 00000100:00100000:2.0:1761315346.124615:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4274 00000100:00100000:2.0:1761315346.124619:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307544000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315346.124640:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307544000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (1320us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315346.124646:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4274 00010000:00010000:1.0:1761315346.139679:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315346.651628:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315346.652279:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2a9d0ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322515264:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315346.652403:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315346.652758:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515264 00000100:00100000:1.0:1761315346.652832:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2681 00000100:00100000:1.0:1761315346.652838:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322515264:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315346.652874:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315346.652966:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322515264:12345-0@lo:41:osp-pre-1-0.0 Request processed in 128us (565us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315346.652973:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2681 00000100:00100000:0.0:1761315346.653763:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2a9d0ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322515264:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315347.163577:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315348.188200:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315349.211627:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315349.211649:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322515392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315349.211720:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315349.211728:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322515328:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315349.211734:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315349.211767:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315349.211842:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515328 00000100:00100000:2.0:1761315349.211892:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2682 00000100:00100000:2.0:1761315349.211897:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322515328:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315349.211927:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315349.211966:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322515328:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (201us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.211972:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2682 00000100:00100000:2.0:1761315349.212010:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322515328:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315349.212607:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515392 00000100:00100000:2.0:1761315349.212695:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322515456:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.212700:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 947 00000100:00080000:3.0:1761315349.212709:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315349.212710:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322515392:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315349.212717:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315349.212721:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315349.212727:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3591 00000100:00080000:3.0:1761315349.212729:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315349.212734:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00002000:00000024:0.0:1761315349.212736:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315349.212754:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000061a066c1 x1846869322515712/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315349.212770:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315349.212774:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315349.212776:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315349.212780:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3591 00000100:00100000:0.0:1761315349.212781:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322515392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (1046us total) trans 0 rc 0/0 00000100:00080000:3.0:1761315349.212784:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315349.212790:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315349.212793:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315349.212795:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 947 00000100:00100000:3.0:1761315349.212807:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000051ddfaba x1846869322515776/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315349.212836:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322515392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315349.212856:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322515712:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315349.212859:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322515840:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.212879:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322515584:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.212896:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315349.212900:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315349.212926:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322515776:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315349.212928:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315349.212930:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315349.212937:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322515648:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.212957:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322515968:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.212963:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315349.212972:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315349.213003:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315349.213007:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515712 00000100:00100000:2.0:1761315349.213013:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515456 00000100:00100000:0.0:1761315349.213031:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315349.213036:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515776 00000100:00100000:0.0:1761315349.213041:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322515520:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213046:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1594 00000100:00100000:2.0:1761315349.213052:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322515456:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.213059:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515968 00000100:00100000:0.0:1761315349.213072:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315349.213095:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322515904:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.213095:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1425 00000100:00100000:3.0:1761315349.213104:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322515968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213117:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322515456:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (327us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213128:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1594 00000100:00100000:0.0:1761315349.213129:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315349.213152:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322515968:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315349.213160:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1425 00000100:00100000:0.0:1761315349.213166:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322515968:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213175:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515840 00000100:00100000:3.0:1761315349.213180:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2099 00000100:00100000:3.0:1761315349.213186:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322515712:12345-0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315349.213195:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315349.213205:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515584 00000100:00100000:0.0:1761315349.213226:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1761315349.213227:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515520 00000100:00100000:0.0:1761315349.213232:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322515776:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213240:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515904 00000100:00100000:2.0:1761315349.213252:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4275 00000100:00100000:2.0:1761315349.213256:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322515840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213284:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322515840:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (388us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213290:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4275 00000100:00100000:2.0:1761315349.213299:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4276 00000100:00100000:2.0:1761315349.213302:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322515584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213326:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2a9d0ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322515584:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213331:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4276 00010000:02020000:3.0:1761315349.213335:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315349.213365:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4277 00000100:00100000:2.0:1761315349.213370:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322515520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.213394:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322515712:12345-0@lo:8:kworker/u8:0.0 Request processed in 207us (499us total) trans 0 rc -19/-19 00010000:02020000:0.0:1761315349.213402:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315349.213405:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2099 00000100:00100000:2.0:1761315349.213407:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322515520:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (334us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213416:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4277 00000100:00100000:3.0:1761315349.213431:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4278 00000100:00100000:3.0:1761315349.213437:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322515904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.213445:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322515776:12345-0@lo:8:kworker/u8:0.0 Request processed in 211us (481us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315349.213453:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2100 00000100:00100000:2.0:1761315349.213466:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322515840:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.213481:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322515904:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (354us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213484:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322515456:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315349.213494:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4278 00000100:00100000:2.0:1761315349.213535:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322515648 00000100:00100000:2.0:1761315349.213563:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2683 00000100:00100000:0.0:1761315349.213569:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322515520:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213570:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322515648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.213587:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322515904:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315349.213602:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322515584:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315349.213618:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322515648:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (645us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315349.213626:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2683 00000100:00080000:1.0:1761315349.213671:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315349.213680:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322515648:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315349.213846:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315349.213851:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322515712:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315349.213864:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315349.213866:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315349.213868:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322515776:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315350.237491:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315351.241639:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315351.243055:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544256 00000100:00100000:0.0:1761315351.243120:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4279 00000100:00100000:0.0:1761315351.243127:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315351.243225:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 96us (1594us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315351.243233:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4279 00000100:00100000:3.0:1761315351.243465:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315351.243849:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315351.244079:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544384 00000100:00100000:1.0:1761315351.244110:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2101 00000100:00100000:1.0:1761315351.244114:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307544384:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315351.244200:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315351.244474:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315351.245786:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544320 00000100:00100000:0.0:1761315351.245849:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4280 00000100:00100000:0.0:1761315351.245854:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307544320:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315351.245927:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307544320:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (1454us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315351.245934:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4280 00000100:00100000:2.0:1761315351.246079:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544192 00000100:00100000:2.0:1761315351.246114:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1595 00000100:00100000:2.0:1761315351.246118:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315351.246151:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (2303us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315351.246157:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1595 00000100:00100000:1.0:1761315351.246422:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307544384:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 2305us (2956us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315351.246431:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2101 00010000:00010000:0.0:1761315351.259883:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315351.771557:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000358802ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322516032:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315351.771625:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315351.771749:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516032 00000100:00100000:3.0:1761315351.774724:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2684 00000100:00100000:3.0:1761315351.774738:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322516032:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315351.774780:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315351.776962:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000358802ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322516032:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315351.777042:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322516032:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2305us (5416us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315351.777052:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2684 00010000:00010000:0.0:1761315352.284476:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315353.307722:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315354.331539:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315354.331551:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315354.331580:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322516096:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315354.331660:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315354.331768:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516096 00000100:00100000:0.0:1761315354.331813:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2685 00000100:00100000:0.0:1761315354.331820:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322516096:12345-0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761315354.331835:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315354.331840:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315354.331842:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315354.331848:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3596 00000004:00000024:0.0:1761315354.331850:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:1.0:1761315354.331851:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315354.331855:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315354.331877:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000422f86fe x1846869322516480/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315354.331887:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322516096:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (228us total) trans 0 rc 0/0 00000100:00080000:1.0:1761315354.331891:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315354.331893:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2685 00000100:00080000:1.0:1761315354.331894:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315354.331895:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315354.331898:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3596 00000100:00080000:1.0:1761315354.331899:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315354.331902:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315354.331912:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000014f5fd29 x1846869322516544/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315354.331928:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a79f4c04 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322516352:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315354.331949:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322516480:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315354.331953:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315354.331972:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322516736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315354.331976:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315354.331992:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322516544:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315354.331993:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315354.332005:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315354.332008:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315354.332010:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322516288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.332028:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315354.332033:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516480 00000100:00100000:0.0:1761315354.332039:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322516672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315354.332050:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516736 00000100:00100000:0.0:1761315354.332056:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315354.332122:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322516160:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315354.332151:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315354.332168:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ef192eeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322516416:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.332186:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315354.333481:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322516224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315354.333527:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315354.333613:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516352 00000100:00100000:1.0:1761315354.333613:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516544 00000100:00100000:0.0:1761315354.333652:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516288 00000100:00100000:0.0:1761315354.333667:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516672 00000100:00100000:0.0:1761315354.333705:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4281 00000100:00100000:0.0:1761315354.333709:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322516352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.333751:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322516352:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (1799us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315354.333757:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4281 00000100:00100000:0.0:1761315354.333769:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4282 00000100:00100000:0.0:1761315354.333772:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000349be38c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322516288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.333795:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000349be38c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322516288:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1767us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315354.333800:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4282 00000100:00100000:0.0:1761315354.333810:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4283 00000100:00100000:1.0:1761315354.333811:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1426 00000100:00100000:0.0:1761315354.333813:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322516672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315354.333815:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322516736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.333837:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322516672:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (1781us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315354.333842:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322516736:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (1849us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315354.333843:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4283 00000100:00100000:1.0:1761315354.333847:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1426 00000100:00100000:1.0:1761315354.333857:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2102 00000100:00100000:0.0:1761315354.333860:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2103 00000100:00100000:1.0:1761315354.333861:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322516480:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315354.333863:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322516544:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315354.334034:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315354.334034:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315354.334070:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322516480:12345-0@lo:8:kworker/u8:2.0 Request processed in 207us (2092us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315354.334076:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2102 00000100:00100000:3.0:1761315354.334123:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322516800:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.334150:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315354.334183:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516160 00000100:00100000:3.0:1761315354.334209:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 948 00000100:00100000:3.0:1761315354.334213:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322516160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315354.334232:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315354.334249:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322516608:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.334263:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322516160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (2112us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315354.334268:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 948 00000100:00100000:2.0:1761315354.334271:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315354.334285:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516800 00000100:00100000:2.0:1761315354.334299:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516608 00000100:00100000:3.0:1761315354.334302:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1427 00000100:00100000:3.0:1761315354.334306:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322516800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315354.334313:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4284 00000100:00100000:2.0:1761315354.334316:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322516608:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.334332:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322516800:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (182us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315354.334337:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1427 00000100:00100000:2.0:1761315354.334356:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322516608:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (71us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315354.334365:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4284 00000100:00100000:2.0:1761315354.334383:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516224 00000100:00100000:3.0:1761315354.334391:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322516160:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315354.334401:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322516800:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315354.334403:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596 00000100:00100000:2.0:1761315354.334406:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322516224:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315354.334432:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322516224:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (905us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315354.334437:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596 00000100:00100000:3.0:1761315354.334455:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1ae97ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322516096:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315354.334465:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322516224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315354.334473:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322516608:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.334506:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516416 00000100:00100000:3.0:1761315354.334549:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2686 00000100:00100000:3.0:1761315354.334552:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322516416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.334738:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322516544:12345-0@lo:8:kworker/u8:2.0 Request processed in 871us (2727us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315354.334745:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2103 00000100:00100000:0.0:1761315354.334785:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322516288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.334793:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c906f54 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322516672:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.334801:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322516736:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315354.334809:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a79f4c04 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322516352:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315354.334911:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315354.334916:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315354.334920:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322516480:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315354.334932:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315354.334935:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315354.334937:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322516544:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315354.334970:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ef192eeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322516416:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315354.335415:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322516416:12345-0@lo:400:kworker/u8:2.0 Request processed in 860us (3226us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315354.335422:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2686 00010000:00010000:0.0:1761315355.355520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315356.362731:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315356.362783:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315356.362826:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315356.363084:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544448 00000100:00100000:2.0:1761315356.363138:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1597 00000100:00100000:2.0:1761315356.363146:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544448:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315356.363215:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544448:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (431us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315356.363222:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1597 00000100:00100000:2.0:1761315356.363497:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315356.363715:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544640 00000100:00100000:0.0:1761315356.363758:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2104 00000100:00100000:0.0:1761315356.363769:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307544640:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315356.363876:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315356.363877:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544576 00000100:00100000:3.0:1761315356.363904:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544512 00000100:00100000:0.0:1761315356.363913:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307544640:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 149us (1191us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315356.363921:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4285 00000100:00100000:3.0:1761315356.363925:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307544576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315356.363929:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2104 00000100:00100000:3.0:1761315356.363957:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307544576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (1132us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315356.363964:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4285 00000100:00100000:3.0:1761315356.363977:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4286 00000100:00100000:3.0:1761315356.363981:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315356.364002:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (507us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315356.364008:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4286 00010000:00010000:2.0:1761315356.380504:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315356.891693:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322516864:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315356.891767:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315356.891904:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516864 00000100:00100000:3.0:1761315356.891966:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2687 00000100:00100000:3.0:1761315356.891973:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322516864:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315356.892004:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315356.894668:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322516864:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2686us (2893us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315356.894685:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2687 00000100:00100000:2.0:1761315356.894779:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322516864:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315357.403508:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315358.427529:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315359.451713:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322516928:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315359.451777:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315359.451840:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516928 00000100:00100000:2.0:1761315359.451878:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2688 00000100:00100000:2.0:1761315359.451883:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322516928:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315359.451912:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315359.451945:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322516928:12345-0@lo:41:osp-pre-0-1.0 Request processed in 61us (169us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315359.451951:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2688 00000100:00100000:2.0:1761315359.451988:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322516928:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315359.452135:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315359.452140:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315359.452144:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315359.452148:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3601 00000100:00080000:3.0:1761315359.452154:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315359.452157:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315359.452173:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cc929884 x1846869322517248/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315359.452187:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315359.452190:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315359.452191:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315359.452194:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3601 00000100:00080000:3.0:1761315359.452195:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315359.452197:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315359.452205:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000abeba833 x1846869322517312/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315359.452266:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322516992:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452299:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452316:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322517376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452339:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452376:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315359.452382:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322517184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452401:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452425:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517376 00000100:00100000:0.0:1761315359.452451:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4287 00000100:00100000:0.0:1761315359.452455:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322517376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452487:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322517376:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (148us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315359.452493:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4287 00000100:00100000:0.0:1761315359.452521:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322517376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452536:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517184 00000100:00100000:0.0:1761315359.452556:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2689 00000100:00100000:0.0:1761315359.452559:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322517184:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452599:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322517184:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (186us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315359.452604:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2689 00000100:00100000:0.0:1761315359.452626:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322517184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452656:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322517248:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452681:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452695:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322517312:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452714:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452741:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322517056:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452761:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452774:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087ee706d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322517440:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452793:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452809:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315359.452814:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322517120:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.452828:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517056 00000100:00100000:0.0:1761315359.452844:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315359.452861:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006e05707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322517504:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.452861:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517440 00000100:00100000:3.0:1761315359.452877:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4288 00000100:00100000:0.0:1761315359.452879:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315359.452881:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322517056:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452903:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517248 00000100:00100000:0.0:1761315359.452919:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517312 00000100:00100000:3.0:1761315359.452923:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322517056:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (161us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315359.452929:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4288 00000100:00100000:0.0:1761315359.452931:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517504 00000100:00100000:3.0:1761315359.452941:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517120 00000100:00100000:0.0:1761315359.452947:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1428 00000100:00100000:0.0:1761315359.452951:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322517504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.452951:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4289 00000100:00100000:3.0:1761315359.452953:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a79f4c04 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322517440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.452975:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322517504:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (95us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315359.452975:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a79f4c04 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322517440:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315359.452980:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1428 00000100:00100000:3.0:1761315359.452980:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4289 00000100:00100000:0.0:1761315359.452991:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2105 00000100:00100000:3.0:1761315359.452991:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4290 00000100:00100000:0.0:1761315359.452994:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322517248:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315359.452994:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049074e99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322517120:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.453012:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049074e99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322517120:12345-0@lo:400:kworker/u8:0.0 Request processed in 18us (169us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315359.453017:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4290 00000100:00100000:3.0:1761315359.453040:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2106 00000100:00100000:3.0:1761315359.453043:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322517312:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315359.453104:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315359.453130:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322517248:12345-0@lo:8:kworker/u8:0.0 Request processed in 135us (449us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315359.453136:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2105 00010000:02020000:3.0:1761315359.453139:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315359.453159:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322517312:12345-0@lo:8:kworker/u8:0.0 Request processed in 115us (445us total) trans 0 rc -19/-19 00000100:00080000:0.0:1761315359.453161:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315359.453164:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315359.453165:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2106 00000100:00100000:0.0:1761315359.453167:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322517248:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315359.453174:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315359.453176:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315359.453178:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322517312:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315359.453195:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005f03965b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322517568:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.453216:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1761315359.453264:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315359.455279:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517568 00000100:00100000:0.0:1761315359.455310:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315359.455323:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024d2fcef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322517632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315359.455330:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1429 00000100:00100000:2.0:1761315359.455337:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322517568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.455366:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322517120:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315359.455386:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006e05707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322517504:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.455396:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315359.455403:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078729988 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322517056:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315359.455413:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322517568:12345-0@lo:400:kworker/u8:0.0 Request processed in 74us (2194us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315359.455422:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1429 00000100:00100000:3.0:1761315359.455424:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087ee706d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322517440:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.455427:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005f03965b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322517568:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.455464:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517632 00000100:00100000:3.0:1761315359.455510:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322516992 00000100:00100000:0.0:1761315359.455511:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 949 00000100:00100000:0.0:1761315359.455517:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322517632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315359.455539:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315359.455549:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1598 00000100:00100000:3.0:1761315359.455555:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322516992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315359.455574:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322517632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (181us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315359.455582:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 949 00000100:00100000:3.0:1761315359.455618:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322516992:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (3317us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315359.455626:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1598 00000100:00100000:0.0:1761315359.455640:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024d2fcef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322517632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315359.455661:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eee489c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322516992:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315360.478472:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315361.480133:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315361.480202:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544704 00000100:00100000:3.0:1761315361.480257:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1599 00000100:00100000:3.0:1761315361.480264:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307544704:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315361.483074:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307544704:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 2802us (2940us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315361.483091:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1599 00000100:00100000:0.0:1761315361.483961:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315361.484000:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315361.484049:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544832 00000100:00100000:0.0:1761315361.484072:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315361.484109:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544768 00000100:00100000:3.0:1761315361.484113:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4291 00000100:00100000:3.0:1761315361.484127:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000613c8591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307544832:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315361.484190:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000613c8591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307544832:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 63us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315361.484262:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544896 00000100:00100000:1.0:1761315361.484287:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2107 00000100:00100000:1.0:1761315361.484291:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307544896:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:1.0:1761315361.484402:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315361.484440:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307544896:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 148us (441us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315361.484447:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2107 00000100:00100000:0.0:1761315361.484956:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4292 00000100:00100000:0.0:1761315361.484962:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544768:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315361.485012:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307544768:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 49us (949us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315361.485019:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4292 00000100:00100000:3.0:1761315361.485839:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4291 00010000:00010000:0.0:1761315361.499467:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315362.011689:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315362.011711:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322517696:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315362.011785:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315362.011843:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517696 00000100:00100000:0.0:1761315362.011898:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2690 00000100:00100000:0.0:1761315362.011904:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322517696:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315362.011937:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315362.011987:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322517696:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (203us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315362.011994:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2690 00000100:00100000:3.0:1761315362.014787:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322517696:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315362.524257:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315363.547503:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315364.571627:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322517760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.571690:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315364.571755:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517760 00000100:00100000:1.0:1761315364.571803:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1600 00000100:00100000:1.0:1761315364.571810:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322517760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.571840:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315364.571847:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322517952:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315364.571863:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322517760:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (172us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315364.571871:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1600 00000100:00100000:3.0:1761315364.571891:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315364.571921:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001bb0e258 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322517760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.571948:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517952 00000100:00100000:1.0:1761315364.571978:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322517888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.571979:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4293 00000100:00100000:3.0:1761315364.571986:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322517952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315364.572021:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315364.572026:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322517952:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (135us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315364.572033:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4293 00000100:00100000:1.0:1761315364.572048:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315364.572056:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315364.572058:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322517824:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.572066:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322518016:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315364.572090:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315364.572098:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315364.572131:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517824 00000100:00100000:3.0:1761315364.572140:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322517952:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315364.572170:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315364.572171:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4294 00000100:00100000:0.0:1761315364.572178:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322517824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315364.572193:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322518080:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315364.572197:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322517888 00000100:00100000:0.0:1761315364.572219:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322517824:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (130us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315364.572224:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518016 00000100:00100000:0.0:1761315364.572227:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4294 00000100:00100000:1.0:1761315364.572233:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315364.572244:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2691 00000100:00100000:3.0:1761315364.572249:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322517888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315364.572261:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2692 00000100:00100000:1.0:1761315364.572266:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322517824:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315364.572269:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322518016:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315364.572286:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322517888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315364.572286:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322517888:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315364.572296:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2691 00000004:00000024:0.0:1761315364.572300:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315364.572319:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518080 00010000:00010000:3.0:1761315364.572332:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315364.572371:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 950 00000100:00100000:0.0:1761315364.572373:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322518016:12345-0@lo:41:osp-pre-0-1.0 Request processed in 101us (272us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315364.572377:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322518016:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315364.572380:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322518080:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315364.572388:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2692 00002000:00000024:1.0:1761315364.572404:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315364.572440:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322518080:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (209us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315364.572448:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 950 00000100:00100000:0.0:1761315364.572477:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322518080:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315364.572495:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de489604 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322518144:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315364.572499:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315364.572504:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315364.572507:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315364.572513:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3607 00000100:00080000:2.0:1761315364.572516:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315364.572522:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315364.572526:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315364.572543:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fb56f29f x1846869322518208/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315364.572571:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315364.572574:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518144 00000100:00080000:2.0:1761315364.572575:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315364.572577:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315364.572581:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3607 00000100:00080000:2.0:1761315364.572583:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315364.572586:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315364.572599:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fc2658e0 x1846869322518272/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315364.572619:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2693 00000100:00100000:2.0:1761315364.572642:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e25f5f92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322518336:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315364.572663:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322518144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572670:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315364.572687:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315364.572692:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322518400:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315364.572707:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322518144:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315364.572715:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2693 00000100:00100000:2.0:1761315364.572719:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315364.572741:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322518208:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572745:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518336 00000100:00100000:2.0:1761315364.572767:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518400 00000100:00100000:0.0:1761315364.572777:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315364.572785:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4295 00000100:00100000:2.0:1761315364.572790:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfb55374 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322518336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315364.572799:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322518272:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572827:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfb55374 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322518336:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (160us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315364.572833:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4295 00000100:00100000:2.0:1761315364.572848:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4296 00000100:00100000:2.0:1761315364.572851:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d62398c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322518400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572874:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d62398c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322518400:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (158us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315364.572880:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4296 00000100:00100000:2.0:1761315364.572904:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322518400:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572911:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e25f5f92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322518336:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.572947:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518208 00000100:00100000:2.0:1761315364.572965:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2108 00000100:00100000:2.0:1761315364.572974:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518208:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315364.573023:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315364.573059:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de489604 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322518144:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315364.573068:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315364.573070:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315364.573076:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a12775 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322518464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.573107:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518208:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (329us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315364.573110:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315364.573118:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2108 00000100:00100000:2.0:1761315364.573130:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518272 00000100:00100000:2.0:1761315364.573146:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518464 00000100:00080000:0.0:1761315364.573149:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315364.573153:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315364.573174:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1430 00000100:00100000:2.0:1761315364.573178:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0629bf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322518464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315364.573201:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0629bf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322518464:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315364.573206:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1430 00000100:00100000:2.0:1761315364.573214:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2109 00000100:00100000:2.0:1761315364.573216:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518272:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315364.573284:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315364.573300:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518272:12345-0@lo:8:kworker/u8:0.0 Request processed in 83us (472us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315364.573305:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2109 00000100:00100000:0.0:1761315364.573380:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322518208:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315364.573395:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315364.573397:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315364.573399:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322518272:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315364.573428:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a12775 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322518464:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315365.595496:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315366.600839:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315366.600890:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315366.600901:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307544960 00000100:00100000:1.0:1761315366.600956:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1601 00000100:00100000:1.0:1761315366.600962:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315366.601038:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307544960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 76us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315366.601045:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1601 00000100:00100000:0.0:1761315366.601202:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545024 00000100:00100000:0.0:1761315366.601243:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4297 00000100:00100000:0.0:1761315366.601247:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307545024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315366.602636:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307545024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1382us (1742us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315366.602650:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4297 00000100:00100000:1.0:1761315366.602871:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315366.603002:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545152 00000100:00100000:2.0:1761315366.603039:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2110 00000100:00100000:2.0:1761315366.603045:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545152:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315366.603131:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315366.603190:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545152:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 143us (319us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315366.603197:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2110 00000100:00100000:3.0:1761315366.606121:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315366.608630:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545088 00000100:00100000:0.0:1761315366.608695:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4298 00000100:00100000:0.0:1761315366.608701:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307545088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315366.608783:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307545088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 81us (2670us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315366.608792:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4298 00010000:00010000:0.0:1761315366.619596:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315367.131799:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315367.131822:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322518528:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315367.131897:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315367.132002:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518528 00000100:00100000:3.0:1761315367.132057:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2694 00000100:00100000:3.0:1761315367.132064:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322518528:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315367.132094:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315367.132145:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322518528:12345-0@lo:41:osp-pre-1-0.0 Request processed in 80us (249us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315367.132152:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2694 00000100:00100000:1.0:1761315367.132277:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322518528:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315367.643833:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315368.667743:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315369.691554:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315369.691602:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315369.691622:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322518592:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315369.691661:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1761315369.691671:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315369.691673:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322518848:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315369.691678:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315369.691682:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315369.691690:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3612 00000100:00100000:3.0:1761315369.691693:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315369.691694:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315369.691702:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315369.691716:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315369.691724:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e36ce75d x1846869322518976/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315369.691744:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322518912:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315369.691749:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315369.691753:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315369.691754:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315369.691759:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3612 00000100:00080000:0.0:1761315369.691761:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315369.691763:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315369.691780:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008d9901b3 x1846869322519040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315369.691821:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322518784:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315369.691929:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322518720:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692044:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322518656:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692073:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315369.692092:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692115:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315369.692131:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315369.692135:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322519296:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692153:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315369.692182:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518848 00000100:00100000:2.0:1761315369.692221:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4299 00000100:00100000:2.0:1761315369.692227:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e25f5f92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322518848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692279:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e25f5f92 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322518848:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (564us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315369.692285:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315369.692290:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e36ce75d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322518976:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692291:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4299 00000100:00100000:0.0:1761315369.692327:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315369.692329:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519232 00000100:00100000:3.0:1761315369.692341:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518592 00000100:00100000:0.0:1761315369.692373:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322519040:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692382:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519296 00000100:00100000:0.0:1761315369.692401:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315369.692409:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518912 00000100:00100000:2.0:1761315369.692414:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518976 00000100:00100000:2.0:1761315369.692431:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519040 00000100:00100000:3.0:1761315369.692440:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2695 00000100:00100000:0.0:1761315369.692442:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1431 00000100:00100000:3.0:1761315369.692448:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322518592:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315369.692450:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cea502e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322519232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.692460:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1432 00000100:00100000:2.0:1761315369.692465:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000802e1776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322519296:12345-0@lo:400:kworker/u8:0.0 00000004:00000024:3.0:1761315369.692477:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315369.692492:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cea502e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322519232:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315369.692500:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1431 00000100:00100000:2.0:1761315369.692527:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000802e1776 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322519296:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315369.692533:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2111 00000100:00100000:3.0:1761315369.692536:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322518592:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (847us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315369.692543:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1432 00000100:00100000:0.0:1761315369.692544:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518976:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692547:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2695 00000100:00100000:2.0:1761315369.692564:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2112 00000100:00100000:3.0:1761315369.692568:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2696 00000100:00100000:2.0:1761315369.692572:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322519040:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692577:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322518912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692604:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322518912:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315369.692609:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2696 00000100:00100000:3.0:1761315369.692635:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315369.692650:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322519168:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692670:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315369.692694:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518784 00000100:00100000:3.0:1761315369.692712:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2697 00000100:00100000:3.0:1761315369.692716:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322518784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692745:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322518784:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (110us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315369.692751:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2697 00000100:00100000:3.0:1761315369.692766:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519168 00000100:00100000:3.0:1761315369.692786:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4300 00000100:00100000:3.0:1761315369.692788:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092b2ea3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322519168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692813:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092b2ea3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322519168:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (143us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315369.692818:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4300 00000100:00100000:3.0:1761315369.692846:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322518912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692859:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322518592:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315369.692885:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322518784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692894:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322519168:0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761315369.692934:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315369.692938:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322519296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.692948:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519232:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315369.692957:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315369.692960:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322518848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315369.692981:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322518976:12345-0@lo:8:kworker/u8:0.0 Request processed in 439us (654us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315369.692989:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2111 00000100:00100000:2.0:1761315369.693001:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322519040:12345-0@lo:8:kworker/u8:0.0 Request processed in 428us (597us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315369.693010:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2112 00000100:00080000:0.0:1761315369.693024:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315369.693033:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518656 00000100:00080000:0.0:1761315369.693034:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315369.693040:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e36ce75d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322518976:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315369.693052:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315369.693055:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315369.693058:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322519040:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315369.693066:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1602 00000100:00100000:3.0:1761315369.693072:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322518656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315369.693127:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322518656:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (1054us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315369.693135:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1602 00000100:00100000:1.0:1761315369.693175:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315369.693175:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322518656:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315369.693212:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322519104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315369.693234:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315369.693256:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076b43207 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322519360:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315369.693281:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315369.693311:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519360 00000100:00100000:1.0:1761315369.693337:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 951 00000100:00100000:1.0:1761315369.693341:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cccdaf22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322519360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315369.693378:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315369.693410:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cccdaf22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322519360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (128us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315369.693420:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 951 00000100:00100000:2.0:1761315369.693428:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322518720 00000100:00100000:1.0:1761315369.693460:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076b43207 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322519360:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315369.693461:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519104 00000100:00100000:2.0:1761315369.693484:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4301 00000100:00100000:2.0:1761315369.693489:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005487ec0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322518720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315369.693494:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4302 00000100:00100000:1.0:1761315369.693499:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000506d5024 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322519104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315369.693540:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005487ec0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322518720:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (369us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315369.693547:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4301 00000100:00100000:1.0:1761315369.693552:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000506d5024 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322519104:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315369.693560:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4302 00000100:00100000:1.0:1761315369.693591:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322518720:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315369.693600:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322519104:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315370.715522:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315371.722759:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315371.722879:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315371.724513:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545216 00000100:00100000:1.0:1761315371.724702:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315371.725341:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315371.725385:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545408 00000100:00100000:1.0:1761315371.725415:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2113 00000100:00100000:1.0:1761315371.725422:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307545408:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315371.725526:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315371.725582:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307545408:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 159us (243us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315371.725590:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2113 00000100:00100000:3.0:1761315371.725719:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1603 00000100:00100000:3.0:1761315371.725724:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307545216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315371.725760:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307545216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (3010us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315371.725767:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1603 00000100:00100000:2.0:1761315371.726434:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545280 00000100:00100000:2.0:1761315371.726481:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545344 00000100:00100000:2.0:1761315371.726518:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4303 00000100:00100000:2.0:1761315371.726523:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076b43207 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307545280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315371.726567:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076b43207 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307545280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 44us (3689us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315371.726575:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4303 00000100:00100000:2.0:1761315371.726589:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4304 00000100:00100000:2.0:1761315371.726593:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001cfc05f5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307545344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315371.726616:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001cfc05f5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307545344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (1918us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315371.726622:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4304 00010000:00010000:3.0:1761315371.739744:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315372.252866:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322519424:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315372.252934:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315372.253174:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519424 00000100:00100000:3.0:1761315372.253798:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2698 00000100:00100000:3.0:1761315372.253805:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322519424:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315372.253842:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315372.253887:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322519424:12345-0@lo:41:osp-pre-1-0.0 Request processed in 81us (954us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315372.253894:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2698 00000100:00100000:0.0:1761315372.254040:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322519424:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315372.763598:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315373.787536:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315374.811587:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322519552:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315374.811644:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c488683 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519488:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315374.811648:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322519616:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315374.811651:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315374.811652:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315374.811658:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315374.811662:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315374.811670:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3617 00000100:00080000:3.0:1761315374.811673:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315374.811678:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315374.811684:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000620e4a68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322519808:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.811692:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315374.811702:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007e10a0c4 x1846869322519936/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315374.811713:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315374.811719:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315374.811721:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519744:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315374.811732:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315374.811736:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315374.811738:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315374.811743:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3617 00000100:00080000:3.0:1761315374.811745:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:1.0:1761315374.811750:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322519872:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315374.811750:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315374.811762:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a421fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322519680:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.811765:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315374.811769:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008135f598 x1846869322520000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315374.811790:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315374.811794:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315374.811806:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322520128:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.811824:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a16ccef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322520064:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.811841:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315374.811853:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e10a0c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322519936:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315374.811855:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519552 00000100:00100000:1.0:1761315374.811861:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519616 00000100:00100000:0.0:1761315374.811878:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315374.811880:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315374.811890:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089c37b13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322520192:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.811895:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008135f598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322520000:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315374.811905:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519744 00000100:00100000:1.0:1761315374.811907:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1604 00000100:00100000:1.0:1761315374.811915:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322519616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.811924:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315374.811926:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519872 00000100:00100000:3.0:1761315374.811947:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2699 00000100:00100000:3.0:1761315374.811952:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322519552:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315374.811977:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2700 00000004:00000024:3.0:1761315374.811980:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315374.811981:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322519616:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (271us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315374.811987:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc7bc95e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322519744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315374.811993:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1604 00000100:00100000:0.0:1761315374.811998:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315374.812041:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2701 00000100:00100000:1.0:1761315374.812047:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2a5a930 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322519872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812049:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519808 00000100:00100000:2.0:1761315374.812051:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc7bc95e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322519744:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315374.812057:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322519552:12345-0@lo:41:osp-pre-0-1.0 Request processed in 101us (407us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315374.812063:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2700 00000100:00100000:3.0:1761315374.812070:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2699 00000100:00100000:0.0:1761315374.812086:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519680 00000100:00100000:1.0:1761315374.812090:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2a5a930 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322519872:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315374.812098:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2701 00000100:00100000:2.0:1761315374.812112:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519744:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812118:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520128 00000100:00100000:3.0:1761315374.812128:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4305 00000100:00100000:1.0:1761315374.812132:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519488 00000100:00100000:3.0:1761315374.812136:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e36ce75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322519808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315374.812162:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 952 00000100:00100000:1.0:1761315374.812166:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000306fc987 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322519488:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315374.812174:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e36ce75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322519808:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (456us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315374.812179:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322519936 00000100:00100000:3.0:1761315374.812183:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4305 00002000:00000024:1.0:1761315374.812186:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315374.812209:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4306 00000100:00100000:2.0:1761315374.812211:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520192 00000100:00100000:0.0:1761315374.812218:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4307 00000100:00100000:3.0:1761315374.812218:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038cef707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322519680:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315374.812224:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000306fc987 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322519488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (533us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315374.812228:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4d7735c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322520128:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812239:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520000 00000100:00100000:1.0:1761315374.812240:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 952 00000100:00100000:3.0:1761315374.812262:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038cef707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322519680:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (468us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315374.812267:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1433 00000100:00100000:2.0:1761315374.812276:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322520192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812280:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4d7735c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322520128:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (438us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315374.812280:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4306 00000100:00100000:1.0:1761315374.812285:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322519872:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812293:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4307 00000100:00100000:1.0:1761315374.812312:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322519616:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315374.812316:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2114 00000100:00100000:2.0:1761315374.812321:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322520192:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (399us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315374.812324:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322519936:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812333:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1433 00000100:00100000:1.0:1761315374.812368:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2115 00000100:00100000:1.0:1761315374.812373:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322520000:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812390:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089c37b13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322520192:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315374.812395:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315374.812402:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322520128:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812409:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315374.812419:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c488683 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322519488:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315374.812439:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a421fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322519680:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812474:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520064 00000100:00100000:0.0:1761315374.812504:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000620e4a68 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322519808:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315374.812516:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322519552:0@lo:41:osp-pre-0-1.0 00010000:02020000:1.0:1761315374.812556:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315374.812598:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322520000:12345-0@lo:8:kworker/u8:2.0 Request processed in 225us (603us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315374.812604:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2115 00000100:00100000:2.0:1761315374.812675:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4308 00000100:00100000:2.0:1761315374.812680:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acbc995e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322520064:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315374.812684:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315374.812688:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315374.812693:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008135f598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322520000:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812719:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acbc995e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322520064:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (312us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315374.812727:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a16ccef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322520064:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315374.812728:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4308 00010000:02020000:3.0:1761315374.813432:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315374.813495:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322519936:12345-0@lo:8:kworker/u8:2.0 Request processed in 1172us (1619us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315374.813504:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2114 00000100:00080000:0.0:1761315374.813585:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315374.813588:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315374.813592:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e10a0c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322519936:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315375.835616:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315376.842883:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315376.842945:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315376.842985:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315376.843031:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315376.843206:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545472 00000100:00100000:0.0:1761315376.843280:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1605 00000100:00100000:0.0:1761315376.843290:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307545472:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315376.843433:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307545472:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 143us (564us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315376.843446:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1605 00000100:00100000:2.0:1761315376.843807:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545536 00000100:00100000:2.0:1761315376.843850:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545600 00000100:00100000:2.0:1761315376.843871:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4309 00000100:00100000:2.0:1761315376.843876:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003049a216 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307545536:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315376.843917:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003049a216 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307545536:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 41us (933us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315376.843924:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4309 00000100:00100000:2.0:1761315376.843937:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4310 00000100:00100000:2.0:1761315376.843940:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbd24176 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307545600:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315376.843964:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbd24176 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307545600:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 23us (934us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315376.843970:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4310 00000100:00100000:3.0:1761315376.844440:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545664 00000100:00100000:3.0:1761315376.844487:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2116 00000100:00100000:3.0:1761315376.844492:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040d28628 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545664:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:3.0:1761315376.844597:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315376.844653:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040d28628 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545664:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 160us (1707us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315376.844659:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2116 00010000:00010000:3.0:1761315376.860620:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315377.371867:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322520256:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315377.371967:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315377.372071:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520256 00000100:00100000:2.0:1761315377.372155:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2702 00000100:00100000:2.0:1761315377.372163:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d52d58b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322520256:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315377.372211:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315377.372472:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d52d58b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322520256:12345-0@lo:41:osp-pre-1-0.0 Request processed in 299us (497us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315377.372481:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2702 00000100:00100000:1.0:1761315377.372501:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322520256:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315377.884074:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315378.907593:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315379.931751:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315379.931775:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322520384:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315379.931804:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315379.931805:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322520448:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315379.931813:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315379.931819:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315379.931827:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3622 00000100:00080000:0.0:1761315379.931831:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315379.931838:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315379.931849:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315379.931871:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005abee3b5 x1846869322520704/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315379.931892:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315379.931895:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315379.931897:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315379.931900:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3622 00000100:00080000:0.0:1761315379.931902:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315379.931905:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315379.931920:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e4a4d687 x1846869322520768/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315379.931923:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322520320:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315379.931987:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520384 00000100:00100000:3.0:1761315379.931997:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315379.931998:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932052:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1606 00000100:00100000:1.0:1761315379.932060:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322520384:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315379.932068:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322520576:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932075:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322520832:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932084:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322520512:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315379.932130:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315379.932138:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932147:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322520384:12345-0@lo:400:kworker/u8:0.0 Request processed in 88us (300us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315379.932154:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1606 00000100:00100000:0.0:1761315379.932160:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa3900a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322520896:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315379.932164:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520448 00000100:00100000:3.0:1761315379.932206:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322520960:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932214:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520576 00000100:00100000:0.0:1761315379.932219:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932232:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322520640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932251:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4311 00000100:00100000:2.0:1761315379.932258:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a3af5c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322520448:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932262:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322520704:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315379.932277:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932282:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315379.932314:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a3af5c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322520448:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (330us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315379.932329:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4311 00000100:00100000:0.0:1761315379.932332:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932334:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521024:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932375:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520960 00000100:00100000:3.0:1761315379.932395:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322520384:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315379.932397:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315379.932408:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4312 00000100:00100000:0.0:1761315379.932410:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322520768:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932419:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bceeb641 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322520576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932454:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315379.932479:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4313 00000100:00100000:1.0:1761315379.932489:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322520960:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932490:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bceeb641 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322520576:12345-0@lo:400:kworker/u8:0.0 Request processed in 70us (365us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315379.932501:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4312 00000100:00100000:3.0:1761315379.932508:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520320 00000100:00100000:0.0:1761315379.932516:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520704 00000100:00100000:2.0:1761315379.932533:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520512 00000100:00100000:1.0:1761315379.932536:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322520960:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (266us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315379.932548:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4313 00000100:00100000:0.0:1761315379.932562:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521024 00000100:00100000:2.0:1761315379.932575:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520640 00000100:00100000:1.0:1761315379.932594:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520768 00000100:00100000:3.0:1761315379.932599:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2703 00000100:00100000:0.0:1761315379.932605:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1434 00000100:00100000:3.0:1761315379.932620:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322520512:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932621:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9328cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521024:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315379.932624:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2117 00000100:00100000:1.0:1761315379.932636:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322520704:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932674:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9328cc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521024:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315379.932681:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1434 00000100:00100000:0.0:1761315379.932700:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2118 00000100:00100000:0.0:1761315379.932704:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322520768:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315379.932767:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315379.932787:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322520512:12345-0@lo:400:kworker/u8:0.0 Request processed in 173us (650us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315379.932799:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2703 00010000:02020000:0.0:1761315379.932802:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315379.932817:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322520704:12345-0@lo:8:kworker/u8:0.0 Request processed in 181us (491us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315379.932824:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322520768:12345-0@lo:8:kworker/u8:0.0 Request processed in 120us (382us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315379.932825:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2117 00000100:00100000:0.0:1761315379.932829:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2118 00000100:00100000:1.0:1761315379.932874:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521024:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.932878:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322520512:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315379.932883:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2704 00000100:00100000:2.0:1761315379.932901:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2705 00000100:00100000:3.0:1761315379.932902:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322520320:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315379.932912:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d72fb3e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322520640:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315379.932914:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315379.932919:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315379.932924:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322520704:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315379.932939:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000004:00000024:3.0:1761315379.932939:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315379.932947:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520896 00000100:00080000:0.0:1761315379.932948:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315379.932953:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322520768:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315379.932964:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d72fb3e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322520640:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (686us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315379.932972:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2705 00000100:00100000:1.0:1761315379.932988:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 953 00000100:00100000:1.0:1761315379.932995:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322520896:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315379.932998:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315379.933013:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322520320:12345-0@lo:41:osp-pre-0-1.0 Request processed in 116us (1026us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315379.933020:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322520640:0@lo:400:kworker/u8:0.0 00002000:00000024:1.0:1761315379.933024:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315379.933029:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2704 00000100:00100000:0.0:1761315379.933033:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087dbe404 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322521088:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315379.933068:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315379.933069:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322520832 00000100:00100000:1.0:1761315379.933081:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322520896:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (869us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315379.933091:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 953 00000100:00100000:0.0:1761315379.933093:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322520448:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315379.933100:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521088 00010000:00010000:0.0:1761315379.933137:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315379.933142:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa3900a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322520896:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315379.933158:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1435 00000100:00100000:3.0:1761315379.933168:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009094b1e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322521088:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315379.933188:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322520576:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315379.933206:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322520960:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315379.933222:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322520320:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315379.933250:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009094b1e4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322521088:12345-0@lo:400:kworker/u8:0.0 Request processed in 82us (182us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315379.933260:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1435 00000100:00100000:0.0:1761315379.933532:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087dbe404 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322521088:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.934159:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4314 00000100:00100000:2.0:1761315379.934165:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322520832:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315379.934248:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322520832:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (1251us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315379.934254:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4314 00000100:00100000:3.0:1761315379.934460:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322520832:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315380.955657:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315381.961174:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315381.963029:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315381.963097:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545728 00000100:00100000:1.0:1761315381.963152:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1607 00000100:00100000:1.0:1761315381.963158:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307545728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315381.963243:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307545728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 83us (2077us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315381.963251:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1607 00000100:00100000:2.0:1761315381.963371:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545792 00000100:00100000:2.0:1761315381.963411:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4315 00000100:00100000:2.0:1761315381.963416:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307545792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315381.963463:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307545792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 46us (439us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315381.963470:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4315 00000100:00100000:1.0:1761315381.967786:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315381.967843:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315381.969065:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545856 00000100:00100000:2.0:1761315381.969165:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4316 00000100:00100000:2.0:1761315381.969175:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa3900a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307545856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315381.969298:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa3900a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307545856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 123us (1523us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315381.969307:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4316 00000100:00100000:3.0:1761315381.969579:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545920 00000100:00100000:3.0:1761315381.969653:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2119 00000100:00100000:3.0:1761315381.969659:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005819e23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545920:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315381.970684:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315381.970758:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005819e23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307545920:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 1097us (2913us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315381.970771:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2119 00010000:00010000:0.0:1761315381.979742:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315382.493908:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315382.493942:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000955fb331 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322521152:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315382.494045:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315382.494200:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521152 00000100:00100000:3.0:1761315382.495007:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2706 00000100:00100000:3.0:1761315382.495018:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322521152:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315382.495069:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315382.495156:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322521152:12345-0@lo:41:osp-pre-1-0.0 Request processed in 139us (1112us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315382.495165:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2706 00000100:00100000:0.0:1761315382.495583:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000955fb331 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322521152:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315383.004033:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315384.027626:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315385.051697:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315385.051709:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315385.051716:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315385.051730:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3627 00000100:00080000:0.0:1761315385.051737:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315385.051748:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315385.051786:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000003046dc2 x1846869322521664/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315385.051809:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315385.051813:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315385.051815:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315385.051830:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3627 00000100:00080000:0.0:1761315385.051832:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315385.051838:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315385.051852:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fbcc5ee5 x1846869322521728/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315385.051907:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c29eca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521216:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315385.052011:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.052051:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322521536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.052078:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.052106:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008b165e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521920:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.052131:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.052152:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:2.0:1761315385.052159:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000955fb331 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322521280:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.053890:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.053942:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322521472:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.053999:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.054017:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322521856:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054035:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.054112:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003046dc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322521664:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054139:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.054155:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fbcc5ee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322521728:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054173:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315385.054210:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521536 00000100:00100000:2.0:1761315385.054279:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2707 00000100:00100000:2.0:1761315385.054286:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026030406 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322521536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054364:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026030406 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322521536:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (2266us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315385.054384:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2707 00000100:00100000:2.0:1761315385.054420:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521472 00000100:00100000:2.0:1761315385.054444:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521856 00000100:00100000:2.0:1761315385.054460:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4317 00000100:00100000:2.0:1761315385.054464:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6b2f1eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322521472:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054490:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6b2f1eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322521472:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (491us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315385.054496:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4317 00000100:00100000:2.0:1761315385.054505:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4318 00000100:00100000:2.0:1761315385.054509:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f547afa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322521856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315385.054538:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f547afa5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322521856:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (503us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315385.054544:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4318 00000100:00100000:3.0:1761315385.054600:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322521344:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315385.054687:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315385.054728:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322521600:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315385.054755:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315385.054786:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315385.054793:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322521408:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315385.054814:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315385.054832:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006ee7382 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322521792:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.055026:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521216 00000100:00100000:1.0:1761315385.055080:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 954 00000100:00100000:1.0:1761315385.055087:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fc39ebc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315385.055128:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315385.055255:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fc39ebc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 166us (3244us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315385.055263:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 954 00000100:00100000:0.0:1761315385.055276:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521920 00000100:00100000:1.0:1761315385.055292:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315385.055304:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a2767f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322521984:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.055308:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521664 00000100:00100000:0.0:1761315385.055330:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521728 00000100:00100000:1.0:1761315385.055341:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315385.055374:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521984 00000100:00100000:1.0:1761315385.055394:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1436 00000100:00100000:1.0:1761315385.055402:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006bf090c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.055456:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006bf090c7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322521920:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (3324us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315385.055464:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1436 00000100:00100000:1.0:1761315385.055480:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2120 00000100:00100000:1.0:1761315385.055484:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba32f26 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322521664:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315385.055583:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315385.055623:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba32f26 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322521664:12345-0@lo:8:kworker/u8:0.0 Request processed in 138us (1483us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315385.055630:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2120 00000100:00100000:1.0:1761315385.055642:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2121 00000100:00100000:1.0:1761315385.055645:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f482099 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322521728:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315385.055721:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315385.055744:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f482099 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322521728:12345-0@lo:8:kworker/u8:0.0 Request processed in 97us (1570us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315385.055749:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2121 00000100:00080000:1.0:1761315385.055792:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315385.055798:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315385.055803:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003046dc2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322521664:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315385.055812:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315385.055814:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315385.055817:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fbcc5ee5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322521728:0@lo:8:kworker/u8:0.0 00010000:00010000:1.0:1761315385.055886:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315385.055901:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521344 00000100:00100000:1.0:1761315385.055939:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4319 00000100:00100000:0.0:1761315385.055948:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1437 00000100:00100000:1.0:1761315385.055948:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322521344:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.055958:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322521984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.055985:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322521344:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (1300us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315385.055992:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4319 00000100:00100000:0.0:1761315385.056012:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322521984:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (671us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315385.056020:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1437 00000100:00100000:1.0:1761315385.056029:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a2767f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322521984:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.056088:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521600 00000100:00100000:0.0:1761315385.056099:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521408 00000100:00100000:1.0:1761315385.056114:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2708 00000100:00100000:1.0:1761315385.056119:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322521600:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315385.056139:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2709 00000100:00100000:0.0:1761315385.056144:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322521408:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056176:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322521408:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (1363us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315385.056182:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2709 00000100:00100000:0.0:1761315385.056244:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322521472:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056258:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322521856:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056270:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008b165e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521920:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056289:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c29eca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322521216:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315385.056303:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322521536:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056396:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521280 00000100:00100000:0.0:1761315385.056437:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1608 00000100:00100000:0.0:1761315385.056442:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322521280:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315385.056480:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322521280:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (2597us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315385.056486:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1608 00000100:00100000:0.0:1761315385.056521:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000955fb331 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322521280:0@lo:400:kworker/u8:0.0 00000004:00000024:1.0:1761315385.057393:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315385.057477:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322521600:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1355us (2719us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315385.057487:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2708 00000100:00100000:3.0:1761315385.057826:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315385.057886:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322521344:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315385.057909:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322521600:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315385.057944:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322521408:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.058656:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322521792 00000100:00100000:1.0:1761315385.058721:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4320 00000100:00100000:1.0:1761315385.058730:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322521792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315385.058834:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322521792:12345-0@lo:400:kworker/u8:0.0 Request processed in 105us (1009us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315385.058841:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4320 00000100:00100000:2.0:1761315385.059781:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006ee7382 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322521792:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315386.080274:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315387.081990:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315387.082269:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546176 00000100:00100000:3.0:1761315387.082333:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2122 00000100:00100000:3.0:1761315387.082341:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546176:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:3.0:1761315387.082942:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315387.083039:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546176:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 699us (2218us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315387.083048:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2122 00000100:00100000:1.0:1761315387.084759:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315387.084798:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315387.084833:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315387.085334:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307545984 00000100:00100000:3.0:1761315387.085428:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1609 00000100:00100000:3.0:1761315387.085436:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307545984:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315387.085514:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307545984:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 77us (761us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315387.085522:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1609 00000100:00100000:0.0:1761315387.085718:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546112 00000100:00100000:0.0:1761315387.085765:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546048 00000100:00100000:0.0:1761315387.085800:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4321 00000100:00100000:0.0:1761315387.085805:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7d99351 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307546112:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315387.085845:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7d99351 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307546112:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 40us (1047us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315387.085852:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4321 00000100:00100000:0.0:1761315387.085864:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4322 00000100:00100000:0.0:1761315387.085867:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003eb47df6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307546048:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315387.085890:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003eb47df6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307546048:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 22us (1058us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315387.085895:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4322 00010000:00010000:0.0:1761315387.103513:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315387.612539:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322522048:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315387.612613:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315387.612691:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522048 00000100:00100000:3.0:1761315387.612766:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2710 00000100:00100000:3.0:1761315387.612774:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbec2d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322522048:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315387.612819:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315387.612915:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbec2d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322522048:12345-0@lo:41:osp-pre-1-0.0 Request processed in 140us (301us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315387.612923:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2710 00000100:00100000:1.0:1761315387.613860:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322522048:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315388.123681:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315389.147680:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315390.173233:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322522176:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315390.173333:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315390.173402:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315390.173409:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322522112:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315390.173441:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315390.173486:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522112 00000100:00100000:0.0:1761315390.173542:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2711 00000100:00100000:0.0:1761315390.173549:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322522112:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315390.173592:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315390.173652:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322522112:12345-0@lo:41:osp-pre-0-1.0 Request processed in 102us (210us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315390.173659:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2711 00000100:00100000:0.0:1761315390.173713:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322522112:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315390.173782:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315390.174534:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522176 00000100:00100000:3.0:1761315390.174591:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 955 00000100:00100000:3.0:1761315390.174597:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322522176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315390.174625:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315390.174686:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322522176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 89us (1355us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315390.174693:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 955 00000100:00100000:3.0:1761315390.174800:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fb5238e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322522240:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315390.174818:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315390.174825:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315390.174830:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315390.174837:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3632 00000100:00080000:2.0:1761315390.174840:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315390.174846:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315390.174849:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315390.174871:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000802e1776 x1846869322522496/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315390.174877:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315390.174886:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef69ebf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322522304:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315390.174890:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315390.174895:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315390.174897:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315390.174901:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3632 00000100:00080000:2.0:1761315390.174904:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315390.174907:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315390.174918:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315390.174921:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b3f71b47 x1846869322522560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315390.174955:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522304 00000100:00100000:2.0:1761315390.174989:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054a12775 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322522624:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315390.174993:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4323 00000100:00100000:3.0:1761315390.174999:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620e4a68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322522304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315390.175033:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315390.175044:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620e4a68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322522304:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (125us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315390.175052:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4323 00000100:00100000:3.0:1761315390.175066:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522624 00000100:00100000:2.0:1761315390.175085:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef69ebf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322522304:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315390.175088:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4324 00000100:00100000:3.0:1761315390.175095:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de489604 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322522624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315390.175108:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315390.175115:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322522688:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315390.175126:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de489604 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322522624:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (94us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315390.175132:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4324 00000100:00100000:2.0:1761315390.175154:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315390.175176:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054a12775 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322522624:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315390.175183:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522688 00000100:00100000:3.0:1761315390.175210:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4325 00000100:00100000:3.0:1761315390.175214:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef69ebf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322522688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315390.175261:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef69ebf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322522688:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (108us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315390.175267:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4325 00000100:00100000:1.0:1761315390.175336:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522240 00000100:00100000:1.0:1761315390.175412:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1610 00000100:00100000:1.0:1761315390.175417:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322522240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175455:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322522240:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (611us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315390.175462:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1610 00000100:00100000:1.0:1761315390.175493:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000802e1776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322522496:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175525:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315390.175544:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3f71b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322522560:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175565:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315390.175591:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d2df127 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322522368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175614:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315390.175629:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322522752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175649:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315390.175663:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315390.175668:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322522432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175693:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315390.175717:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522368 00000100:00100000:1.0:1761315390.175740:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4326 00000100:00100000:1.0:1761315390.175743:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000176f6c62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322522368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175773:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000176f6c62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322522368:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (159us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315390.175779:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4326 00000100:00100000:1.0:1761315390.175806:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d2df127 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322522368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175828:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522432 00000100:00100000:1.0:1761315390.175850:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2712 00000100:00100000:1.0:1761315390.175854:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009716890d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322522432:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175887:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009716890d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322522432:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (193us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315390.175892:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2712 00000100:00100000:1.0:1761315390.175914:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322522432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.175934:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522496 00000100:00100000:1.0:1761315390.175951:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522560 00000100:00100000:1.0:1761315390.175968:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522752 00000100:00100000:1.0:1761315390.175986:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1438 00000100:00100000:1.0:1761315390.175990:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7076cb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322522752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315390.176024:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7076cb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322522752:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315390.176030:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1438 00000100:00100000:1.0:1761315390.176041:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2123 00000100:00100000:1.0:1761315390.176043:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000498385af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322522496:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315390.176136:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315390.176170:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000498385af pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322522496:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (644us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315390.176176:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2123 00000100:00100000:1.0:1761315390.176186:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2124 00000100:00100000:1.0:1761315390.176189:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000b2151c9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322522560:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315390.176261:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315390.176284:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000b2151c9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322522560:12345-0@lo:8:kworker/u8:0.0 Request processed in 94us (719us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315390.176290:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2124 00000100:00080000:1.0:1761315390.176321:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315390.176325:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315390.176329:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000802e1776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322522496:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315390.176338:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315390.176340:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315390.176380:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3f71b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322522560:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315390.176412:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322522752:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315390.176667:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322522688:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315390.176680:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fb5238e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322522240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315390.176794:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322522176:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1761315391.197360:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315392.205862:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315392.205911:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315392.205949:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315392.205978:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315392.206058:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546240 00000100:00100000:3.0:1761315392.206108:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1611 00000100:00100000:3.0:1761315392.206115:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307546240:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315392.206193:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307546240:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 77us (243us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315392.206200:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1611 00000100:00100000:0.0:1761315392.207113:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546432 00000100:00100000:0.0:1761315392.207238:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2125 00000100:00100000:0.0:1761315392.207244:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307546432:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:0.0:1761315392.207341:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315392.207437:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546304 00000100:00100000:3.0:1761315392.207466:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546368 00000100:00100000:3.0:1761315392.207483:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4327 00000100:00100000:3.0:1761315392.207487:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c29eca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307546304:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315392.207553:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c29eca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307546304:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 64us (1698us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315392.207560:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4327 00000100:00100000:3.0:1761315392.207572:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4328 00000100:00100000:3.0:1761315392.207576:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040d28628 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307546368:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315392.207599:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040d28628 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307546368:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 22us (1621us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315392.207604:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4328 00000100:00100000:0.0:1761315392.208385:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307546432:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 1139us (2473us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315392.208393:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2125 00010000:00010000:3.0:1761315392.222897:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315392.731657:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322522816:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315392.731734:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315392.731887:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522816 00000100:00100000:0.0:1761315392.731941:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2713 00000100:00100000:0.0:1761315392.731946:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fb5238e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322522816:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315392.731977:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315392.732025:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fb5238e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322522816:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (291us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315392.732035:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2713 00000100:00100000:2.0:1761315392.732110:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322522816:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315393.243554:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315394.269846:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315395.291544:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315395.291546:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322522880:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315395.291620:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.291653:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315395.291661:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e9328cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322522944:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315395.291672:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315395.291678:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315395.291682:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315395.291687:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3637 00000100:00080000:1.0:1761315395.291690:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:0.0:1761315395.291693:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315395.291697:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315395.291716:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322523136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.291717:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008c6ba408 x1846869322523264/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315395.291736:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315395.291739:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315395.291741:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315395.291744:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3637 00000100:00080000:1.0:1761315395.291747:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:0.0:1761315395.291750:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315395.291752:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315395.291763:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b68078e1 x1846869322523328/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315395.291775:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315395.291783:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322523200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.291803:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322523264:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315395.291810:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.291835:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322523520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.291842:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315395.291865:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322523328:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315395.291871:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315395.291893:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.291916:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523136 00000100:00100000:1.0:1761315395.291947:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523264 00000100:00100000:0.0:1761315395.291959:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4329 00000100:00100000:0.0:1761315395.291965:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322523136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.291971:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523520 00000100:00100000:1.0:1761315395.291991:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523328 00000100:00100000:1.0:1761315395.292005:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1439 00000100:00100000:1.0:1761315395.292009:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322523520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315395.292035:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322523136:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (287us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315395.292043:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4329 00000100:00100000:1.0:1761315395.292047:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322523520:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.292054:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1439 00000100:00100000:0.0:1761315395.292069:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2126 00000100:00100000:1.0:1761315395.292077:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2127 00000100:00100000:0.0:1761315395.292078:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322523264:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315395.292085:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322523328:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315395.292610:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315395.292632:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315395.292649:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322523264:12345-0@lo:8:kworker/u8:2.0 Request processed in 573us (809us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315395.292658:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2126 00000100:00100000:1.0:1761315395.292669:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322523328:12345-0@lo:8:kworker/u8:2.0 Request processed in 583us (776us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315395.292677:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2127 00000100:00100000:0.0:1761315395.292704:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322523520:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315395.292718:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cc929884 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322523136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.292720:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c488683 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322523008:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.292750:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.292774:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522944 00000100:00100000:1.0:1761315395.292774:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322523392:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.292803:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.292807:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1612 00000100:00100000:0.0:1761315395.292813:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322522944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.292825:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315395.292832:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322523072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315395.292858:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322522944:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (1167us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.292866:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.292871:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1612 00000100:00100000:1.0:1761315395.292888:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322523456:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315395.292904:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523200 00000100:00100000:1.0:1761315395.292912:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315395.292932:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523072 00000100:00100000:1.0:1761315395.292946:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2714 00000100:00100000:1.0:1761315395.292955:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322523200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315395.292956:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2715 00000100:00100000:0.0:1761315395.292964:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322523072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315395.293311:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523008 00000100:00100000:2.0:1761315395.293363:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523392 00000100:00100000:2.0:1761315395.293382:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523456 00000100:00100000:2.0:1761315395.293396:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4330 00000100:00100000:2.0:1761315395.293400:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322523008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293483:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322523200:12345-0@lo:400:kworker/u8:2.0 Request processed in 531us (1674us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.293489:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2714 00000100:00100000:1.0:1761315395.293536:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322523200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293545:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e9328cc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322522944:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293580:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4331 00000100:00100000:1.0:1761315395.293583:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322523392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293614:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322523392:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (813us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.293619:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4331 00000100:00100000:1.0:1761315395.293630:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4332 00000100:00100000:1.0:1761315395.293633:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322523456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293654:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322523456:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (743us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.293658:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4332 00000100:00100000:1.0:1761315395.293678:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322523456:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315395.293686:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322523392:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315395.293717:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315395.293721:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315395.293724:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322523264:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315395.293731:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315395.293734:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315395.293736:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322523328:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315395.294748:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322522880 00000100:00100000:3.0:1761315395.294796:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 956 00000100:00100000:3.0:1761315395.294801:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322522880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315395.294832:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315395.294889:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322522880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (3271us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315395.294896:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 956 00000100:00100000:3.0:1761315395.294985:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322522880:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315395.294999:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315395.295013:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a421fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322523584:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315395.295043:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315395.295076:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523584 00000100:00100000:1.0:1761315395.295157:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2716 00000100:00100000:1.0:1761315395.295162:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322523584:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315395.295186:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315395.295222:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322523584:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315395.295228:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2716 00000100:00100000:3.0:1761315395.296694:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a421fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322523584:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315395.306103:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322523072:12345-0@lo:400:kworker/u8:2.0 Request processed in 13131us (13232us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315395.306118:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2715 00000100:00100000:1.0:1761315395.306683:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322523072:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315395.321423:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322523008:12345-0@lo:400:kworker/u8:2.0 Request processed in 28004us (28655us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315395.321438:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4330 00000100:00100000:1.0:1761315395.322666:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c488683 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322523008:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315396.315670:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315397.320654:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315397.320700:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315397.320731:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315397.320761:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315397.320808:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546560 00000100:00100000:1.0:1761315397.320871:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546624 00000100:00100000:1.0:1761315397.320898:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4333 00000100:00100000:1.0:1761315397.320904:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a421fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307546560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315397.320980:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a421fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307546560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315397.320988:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4333 00000100:00100000:1.0:1761315397.321004:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4334 00000100:00100000:1.0:1761315397.321008:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307546624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315397.321032:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307546624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315397.321037:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4334 00000100:00100000:0.0:1761315397.321076:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546496 00000100:00100000:0.0:1761315397.321108:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1613 00000100:00100000:0.0:1761315397.321112:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307546496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315397.321140:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307546496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 27us (440us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315397.321148:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1613 00000100:00100000:2.0:1761315397.321766:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546688 00000100:00100000:2.0:1761315397.321800:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2128 00000100:00100000:2.0:1761315397.321804:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546688:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315397.321892:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315397.321954:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546688:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 148us (1192us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315397.321961:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2128 00010000:00010000:1.0:1761315397.339566:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315397.851570:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315397.851591:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002aae74d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322523648:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315397.851651:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315397.851707:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523648 00000100:00100000:0.0:1761315397.851764:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2717 00000100:00100000:0.0:1761315397.851771:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054a12775 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322523648:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315397.851810:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315397.851869:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054a12775 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322523648:12345-0@lo:41:osp-pre-1-0.0 Request processed in 97us (217us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315397.851876:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2717 00000100:00100000:2.0:1761315397.851994:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002aae74d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322523648:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315398.367459:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315399.387492:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315400.411622:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315400.411646:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322523840:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315400.411648:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315400.411655:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315400.411659:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315400.411665:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3642 00000100:00080000:2.0:1761315400.411667:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315400.411673:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315400.411694:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000802e1776 x1846869322524096/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315400.411708:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315400.411712:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315400.411714:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315400.411718:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315400.411720:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3642 00000100:00080000:2.0:1761315400.411724:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315400.411727:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315400.411738:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cea502e5 x1846869322524160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315400.411754:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322523776:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315400.411787:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315400.411792:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322523712:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315400.411811:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d2df127 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322524224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.411832:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315400.411843:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315400.411855:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322523968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.411865:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000802e1776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322524096:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315400.411871:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322524480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315400.411886:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315400.411905:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315400.411908:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9dc697d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322524352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315400.411911:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315400.411928:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322524160:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315400.411938:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315400.411947:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3f71b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322524032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.411948:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315400.411975:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524096 00000100:00100000:1.0:1761315400.412000:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524352 00000100:00100000:1.0:1761315400.412022:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1440 00000100:00100000:1.0:1761315400.412028:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322524352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315400.412057:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523712 00000100:00100000:0.0:1761315400.412096:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1614 00000100:00100000:0.0:1761315400.412100:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322523712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412341:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315400.412375:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322523904:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.412377:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322524352:12345-0@lo:400:kworker/u8:0.0 Request processed in 346us (430us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.412386:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1440 00000100:00100000:1.0:1761315400.412404:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2129 00000100:00100000:1.0:1761315400.412412:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322524096:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412413:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315400.412433:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fafad97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322524288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412453:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315400.412483:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9dc697d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322524352:0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315400.412500:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315400.412512:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523840 00000100:00100000:1.0:1761315400.412532:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322524096:12345-0@lo:8:kworker/u8:0.0 Request processed in 122us (628us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315400.412540:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2129 00000100:00100000:2.0:1761315400.412542:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524224 00000100:00100000:2.0:1761315400.412566:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523968 00000100:00100000:1.0:1761315400.412575:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524288 00000100:00100000:2.0:1761315400.412582:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4335 00000100:00100000:2.0:1761315400.412586:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf9b4b30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322523840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.412613:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4336 00000100:00100000:2.0:1761315400.412622:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf9b4b30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322523840:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (906us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.412623:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322524224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412635:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4335 00000100:00100000:2.0:1761315400.412649:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4337 00000100:00100000:2.0:1761315400.412653:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322523968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.412666:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322524224:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (824us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.412674:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4336 00000100:00100000:2.0:1761315400.412690:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322523968:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (804us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.412693:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4338 00000100:00100000:1.0:1761315400.412701:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322524288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412702:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4337 00000100:00100000:2.0:1761315400.412731:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315400.412739:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078729988 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322524288:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.412746:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4338 00000100:00080000:2.0:1761315400.412757:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315400.412761:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315400.412765:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000802e1776 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322524096:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315400.412786:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524160 00000100:00100000:2.0:1761315400.412807:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2130 00000100:00100000:2.0:1761315400.412812:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322524160:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315400.412832:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523776 00000100:00100000:1.0:1761315400.412855:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322523904 00000100:00100000:1.0:1761315400.412867:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2718 00000100:00100000:1.0:1761315400.412870:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322523776:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315400.412890:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00010000:02020000:2.0:1761315400.412891:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315400.412934:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322523712:12345-0@lo:400:kworker/u8:0.0 Request processed in 831us (1103us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315400.412943:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322524160:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (212us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315400.412945:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1614 00000100:00100000:1.0:1761315400.412946:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322523776:12345-0@lo:41:osp-pre-0-1.0 Request processed in 72us (1158us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315400.412957:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2130 00000100:00100000:1.0:1761315400.412958:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2718 00000100:00100000:1.0:1761315400.412979:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2719 00000100:00100000:1.0:1761315400.412985:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322523904:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315400.412997:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315400.413001:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315400.413006:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322524160:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315400.413021:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eee489c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322523904:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (612us total) trans 0 rc 0/0 00010000:00010000:0.0:1761315400.413022:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315400.413029:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2719 00000100:00100000:2.0:1761315400.413038:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322523904:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.413051:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fafad97 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322524288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.413062:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322523712:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.413075:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524480 00000100:00100000:2.0:1761315400.413075:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322523968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315400.413104:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 957 00000100:00100000:1.0:1761315400.413110:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322524480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315400.413125:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315400.413156:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322524480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (1250us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315400.413162:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 957 00000100:00100000:3.0:1761315400.413167:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315400.413195:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322524416:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315400.413217:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315400.413249:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322524480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315400.413260:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d2df127 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322524224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.413273:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524416 00000100:00100000:3.0:1761315400.413274:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc2658e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322523840:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315400.413289:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322523776:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315400.413316:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1441 00000100:00100000:2.0:1761315400.413321:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322524416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315400.413379:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043048615 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322524416:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315400.413384:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1441 00000100:00100000:3.0:1761315400.413541:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000696bc59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322524416:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315400.413823:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524032 00000100:00100000:0.0:1761315400.413909:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2720 00000100:00100000:0.0:1761315400.413913:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000883ca932 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322524032:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315400.414009:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000883ca932 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322524032:12345-0@lo:400:kworker/u8:0.0 Request processed in 94us (845us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315400.414017:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2720 00000100:00100000:3.0:1761315400.414245:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3f71b47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322524032:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315401.436129:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315402.440373:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315402.440429:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315402.441675:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546816 00000100:00100000:1.0:1761315402.441748:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4339 00000100:00100000:1.0:1761315402.441755:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307546816:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315402.441853:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307546816:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 98us (1520us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315402.441862:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4339 00000100:00100000:2.0:1761315402.442999:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546752 00000100:00100000:2.0:1761315402.443040:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1615 00000100:00100000:2.0:1761315402.443045:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307546752:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315402.443088:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307546752:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 42us (2660us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315402.443096:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1615 00000100:00100000:3.0:1761315402.447726:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315402.447997:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546880 00000100:00100000:0.0:1761315402.448053:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4340 00000100:00100000:0.0:1761315402.448060:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307546880:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315402.448137:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307546880:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 77us (420us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315402.448145:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4340 00000100:00100000:0.0:1761315402.449120:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315402.449189:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307546944 00000100:00100000:1.0:1761315402.449409:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2131 00000100:00100000:1.0:1761315402.449414:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546944:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:1.0:1761315402.449506:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315402.449558:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307546944:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 143us (442us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315402.449565:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2131 00010000:00010000:0.0:1761315402.460521:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315402.971714:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e896943 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322524544:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315402.971782:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315402.972545:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524544 00000100:00100000:3.0:1761315402.972596:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2721 00000100:00100000:3.0:1761315402.972602:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322524544:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315402.972633:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315402.976414:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322524544:12345-0@lo:41:osp-pre-1-0.0 Request processed in 3801us (4623us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315402.976431:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2721 00000100:00100000:0.0:1761315402.976748:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e896943 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322524544:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315403.484482:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315404.507602:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315405.531541:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322524608:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315405.531605:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315405.531652:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524608 00000100:00100000:2.0:1761315405.531773:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2722 00000100:00100000:2.0:1761315405.531780:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322524608:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315405.531809:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315405.531848:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322524608:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (243us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315405.531855:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2722 00000100:00100000:2.0:1761315405.531892:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322524608:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315405.532070:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315405.532082:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322524672:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315405.532117:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315405.532123:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1761315405.532125:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315405.532128:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315405.532134:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3647 00000100:00080000:1.0:1761315405.532137:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315405.532141:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315405.532147:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322524736:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532172:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005abee3b5 x1846869322524928/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315405.532185:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315405.532188:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315405.532189:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315405.532192:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3647 00000100:00080000:1.0:1761315405.532193:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315405.532196:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315405.532207:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000a4fe2a6 x1846869322524992/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315405.532248:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322525056:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532308:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532330:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315405.532335:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322525120:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532383:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532408:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525056 00000100:00100000:1.0:1761315405.532433:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525120 00000100:00100000:1.0:1761315405.532449:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4341 00000100:00100000:1.0:1761315405.532453:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c488683 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322525056:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532488:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c488683 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322525056:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.532494:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4341 00000100:00100000:1.0:1761315405.532506:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4342 00000100:00100000:1.0:1761315405.532510:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9328cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322525120:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532535:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9328cc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322525120:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.532541:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4342 00000100:00100000:1.0:1761315405.532571:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b68078e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322525120:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532581:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322525056:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532599:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322524864:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532619:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532632:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322525248:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532651:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532663:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315405.532668:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322524800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532687:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532700:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322525184:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532720:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315405.532742:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525248 00000100:00100000:1.0:1761315405.532764:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525184 00000100:00100000:1.0:1761315405.532779:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1442 00000100:00100000:1.0:1761315405.532783:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322525248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532811:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322525248:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.532816:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1442 00000100:00100000:1.0:1761315405.532826:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1443 00000100:00100000:1.0:1761315405.532828:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322525184:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532846:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322525184:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (126us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.532850:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1443 00000100:00100000:1.0:1761315405.532883:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524800 00000100:00100000:1.0:1761315405.532903:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4343 00000100:00100000:1.0:1761315405.532906:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322524800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532933:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322524800:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (246us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.532939:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4343 00000100:00100000:1.0:1761315405.532963:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322524800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532975:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322525184:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.532983:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322525248:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.533021:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524672 00000100:00100000:1.0:1761315405.533042:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1616 00000100:00100000:1.0:1761315405.533046:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322524672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.533073:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322524672:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (950us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.533079:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1616 00000100:00100000:1.0:1761315405.533108:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524864 00000100:00100000:1.0:1761315405.533125:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2723 00000100:00100000:1.0:1761315405.533128:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322524864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315405.533158:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322524864:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (538us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315405.533163:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2723 00000100:00100000:1.0:1761315405.533185:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322524864:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315405.533541:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315405.533677:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315405.533711:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322524672:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315405.533726:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315405.533732:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e896943 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322525312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315405.533760:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315405.533784:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524736 00000100:00100000:3.0:1761315405.533793:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525312 00000100:00100000:0.0:1761315405.533819:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4344 00000100:00100000:3.0:1761315405.533823:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 958 00000100:00100000:0.0:1761315405.533827:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322524736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315405.533842:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322525312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315405.533868:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315405.533872:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322524736:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (194us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315405.533880:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4344 00000100:00100000:3.0:1761315405.533903:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322525312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (142us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315405.533910:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 958 00000100:00100000:3.0:1761315405.533939:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e896943 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322525312:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315405.533949:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322524736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315405.534155:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322524928:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315405.534187:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315405.534214:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322524992:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315405.534235:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315405.534284:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524928 00000100:00100000:2.0:1761315405.534305:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322524992 00000100:00100000:2.0:1761315405.534319:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2132 00000100:00100000:2.0:1761315405.534323:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322524928:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315405.534435:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315405.534468:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322524928:12345-0@lo:8:kworker/u8:0.0 Request processed in 144us (281us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315405.534475:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2132 00000100:00100000:2.0:1761315405.534487:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2133 00000100:00100000:2.0:1761315405.534490:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322524992:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315405.534568:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315405.534592:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322524992:12345-0@lo:8:kworker/u8:0.0 Request processed in 101us (357us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315405.534598:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2133 00000100:00080000:2.0:1761315405.534633:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315405.534637:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315405.534640:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322524928:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315405.534648:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315405.534651:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315405.534653:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322524992:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315406.556320:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315407.565888:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315407.567172:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315407.567620:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547072 00000100:00100000:0.0:1761315407.567663:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4345 00000100:00100000:0.0:1761315407.567669:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc2658e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547072:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:0.0:1761315407.567746:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc2658e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547072:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 76us (1866us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315407.567754:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4345 00000100:00100000:2.0:1761315407.568511:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315407.568547:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315407.569985:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547136 00000100:00100000:3.0:1761315407.570048:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4346 00000100:00100000:3.0:1761315407.570054:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d2df127 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547136:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315407.570119:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d2df127 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547136:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 65us (1611us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315407.570126:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4346 00000100:00100000:3.0:1761315407.570151:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547200 00000100:00100000:3.0:1761315407.570166:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2134 00000100:00100000:3.0:1761315407.570171:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb56f29f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547200:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00010000:02020000:3.0:1761315407.570294:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315407.570319:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb56f29f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547200:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 147us (1772us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315407.570324:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2134 00000100:00100000:1.0:1761315407.571109:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547008 00000100:00100000:1.0:1761315407.571156:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1617 00000100:00100000:1.0:1761315407.571161:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307547008:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315407.571209:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307547008:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 47us (4041us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315407.571215:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1617 00010000:00010000:3.0:1761315407.582407:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315408.091556:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315408.091576:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ca69bbd1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322525376:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315408.091637:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315408.091734:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525376 00000100:00100000:0.0:1761315408.091787:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2724 00000100:00100000:0.0:1761315408.091792:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322525376:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315408.091824:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315408.091875:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322525376:12345-0@lo:41:osp-pre-1-0.0 Request processed in 79us (235us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315408.091882:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2724 00000100:00100000:2.0:1761315408.092007:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ca69bbd1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322525376:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315408.603582:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315409.627489:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315410.651555:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315410.651561:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315410.651565:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315410.651573:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3653 00000100:00080000:1.0:1761315410.651576:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315410.651581:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315410.651600:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e4a4d687 x1846869322525760/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315410.651613:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315410.651616:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315410.651617:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315410.651620:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3653 00000100:00080000:1.0:1761315410.651622:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315410.651624:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315410.651633:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d0629bf1 x1846869322525824/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315410.651858:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322525568:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.651921:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315410.651947:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322525952:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.651971:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315410.651993:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315410.651997:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322525632:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.652016:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315410.652031:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ddb024e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526016:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.652049:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315410.652076:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525952 00000100:00100000:0.0:1761315410.652107:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525632 00000100:00100000:0.0:1761315410.652405:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4347 00000100:00100000:0.0:1761315410.652413:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322525952:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.652457:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322525952:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (486us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315410.652464:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4347 00000100:00100000:0.0:1761315410.652480:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4348 00000100:00100000:0.0:1761315410.652483:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e896943 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322525632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.652501:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e896943 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322525632:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (485us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315410.652506:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4348 00000100:00100000:0.0:1761315410.652534:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322525632:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.652542:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322525952:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.652644:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322525760:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315410.652670:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.652681:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322525824:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315410.652693:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.652717:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526016 00000100:00100000:2.0:1761315410.652737:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525760 00000100:00100000:2.0:1761315410.652748:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525824 00000100:00100000:2.0:1761315410.652757:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1444 00000100:00100000:2.0:1761315410.652760:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322526016:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.652784:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322526016:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (735us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.652789:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1444 00000100:00100000:2.0:1761315410.652798:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2135 00000100:00100000:2.0:1761315410.652800:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011d94811 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322525760:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315410.652882:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315410.652911:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011d94811 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322525760:12345-0@lo:8:kworker/u8:2.0 Request processed in 109us (241us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315410.652917:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2135 00000100:00100000:2.0:1761315410.652927:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2136 00000100:00100000:2.0:1761315410.652930:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cea502e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322525824:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315410.653001:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315410.653022:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cea502e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322525824:12345-0@lo:8:kworker/u8:2.0 Request processed in 90us (328us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315410.653027:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2136 00000100:00080000:2.0:1761315410.653058:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315410.653062:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315410.653065:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322525760:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315410.653072:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315410.653074:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315410.653076:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322525824:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653099:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322525504:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653122:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.653135:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322525888:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653155:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.653170:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:2.0:1761315410.653176:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322525440:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653214:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.653264:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322525696:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653285:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.653299:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322526080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653329:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315410.653376:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525568 00000100:00100000:2.0:1761315410.653397:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525696 00000100:00100000:2.0:1761315410.653409:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2725 00000100:00100000:2.0:1761315410.653413:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322525568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653438:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322525568:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (1519us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.653443:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2725 00000100:00100000:2.0:1761315410.653452:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2726 00000100:00100000:2.0:1761315410.653455:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322525696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653476:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322525696:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (191us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.653481:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2726 00000100:00100000:2.0:1761315410.653493:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526080 00000100:00100000:2.0:1761315410.653509:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1445 00000100:00100000:2.0:1761315410.653512:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322526080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653534:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322526080:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (206us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.653538:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1445 00000100:00100000:2.0:1761315410.653552:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525504 00000100:00100000:2.0:1761315410.653569:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525888 00000100:00100000:2.0:1761315410.653581:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4349 00000100:00100000:2.0:1761315410.653584:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000802e1776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322525504:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653607:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000802e1776 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322525504:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (485us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.653612:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4349 00000100:00100000:2.0:1761315410.653621:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4350 00000100:00100000:2.0:1761315410.653625:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322525888:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653646:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322525888:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (492us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315410.653651:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4350 00000100:00100000:2.0:1761315410.653699:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322525696:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653708:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322526080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653715:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322525504:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.653722:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322525888:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761315410.653829:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315410.653939:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ddb024e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526016:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315410.653949:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322525568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315410.654621:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322526144:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315410.654663:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315410.654720:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526144 00000100:00100000:1.0:1761315410.654761:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2727 00000100:00100000:1.0:1761315410.654765:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322526144:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315410.654789:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315410.654837:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322526144:12345-0@lo:41:osp-pre-0-1.0 Request processed in 71us (174us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315410.654843:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2727 00000100:00100000:0.0:1761315410.654918:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322526144:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315410.655380:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322525440 00000100:00100000:3.0:1761315410.655410:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1618 00000100:00100000:3.0:1761315410.655416:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322525440:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315410.655466:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322525440:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (2251us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315410.655473:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1618 00000100:00100000:0.0:1761315410.655483:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322525440:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315410.655506:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315410.655518:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322526208:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315410.655547:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315410.655597:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526208 00000100:00100000:3.0:1761315410.656420:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 959 00000100:00100000:3.0:1761315410.656426:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322526208:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315410.656455:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315410.656509:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322526208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (962us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315410.656516:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 959 00000100:00100000:1.0:1761315410.656609:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322526208:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315411.675549:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315412.686567:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315412.686628:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315412.686662:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315412.686779:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315412.686934:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547392 00000100:00100000:3.0:1761315412.686997:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547328 00000100:00100000:3.0:1761315412.687025:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4351 00000100:00100000:3.0:1761315412.687033:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315412.687199:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 166us (571us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315412.687207:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4351 00000100:00100000:3.0:1761315412.687239:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4352 00000100:00100000:3.0:1761315412.687242:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315412.687268:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (605us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315412.687274:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4352 00000100:00100000:1.0:1761315412.687501:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547264 00000100:00100000:1.0:1761315412.687542:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1619 00000100:00100000:1.0:1761315412.687546:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307547264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315412.687581:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307547264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (1027us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315412.687588:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1619 00000100:00100000:0.0:1761315412.687749:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547456 00000100:00100000:0.0:1761315412.687788:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2137 00000100:00100000:0.0:1761315412.687793:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547456:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315412.687887:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315412.687920:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547456:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 126us (1141us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315412.687927:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2137 00010000:00010000:0.0:1761315412.699581:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315413.215627:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322526272:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315413.215723:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315413.215814:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526272 00000100:00100000:1.0:1761315413.217192:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2728 00000100:00100000:1.0:1761315413.217203:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060e3fc1e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322526272:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315413.217270:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315413.217371:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060e3fc1e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322526272:12345-0@lo:41:osp-pre-1-0.0 Request processed in 169us (1649us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315413.217380:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2728 00000100:00100000:2.0:1761315413.217429:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322526272:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315413.724718:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315414.747508:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315415.772123:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315415.772131:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315415.772134:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315415.772142:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3658 00000100:00080000:3.0:1761315415.772145:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315415.772150:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315415.772203:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bc9be46d x1846869322526656/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315415.772219:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315415.772222:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315415.772223:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315415.772226:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3658 00000100:00080000:3.0:1761315415.772228:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315415.772231:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315415.772242:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fe1c4fae x1846869322526720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315415.772257:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322526400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.772292:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.772316:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315415.772317:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.772426:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322526336:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315415.772448:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526400 00000100:00100000:0.0:1761315415.772489:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4353 00000100:00100000:0.0:1761315415.772495:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322526400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.773860:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d77d078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322526848:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.773896:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315415.773922:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315415.773927:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322526528:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.773949:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315415.773963:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.773982:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315415.774017:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526848 00000100:00100000:3.0:1761315415.774044:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526528 00000100:00100000:3.0:1761315415.774061:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4354 00000100:00100000:3.0:1761315415.774065:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322526848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315415.774102:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322526848:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315415.774108:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4354 00000100:00100000:2.0:1761315415.774294:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322526784:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.774319:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315415.774329:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000006ee7382 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322527040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315415.774363:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315415.774398:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526784 00000100:00100000:1.0:1761315415.774571:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.774592:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f31517f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322526592:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315415.774593:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322526400:12345-0@lo:400:kworker/u8:0.0 Request processed in 2096us (2277us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315415.774599:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4353 00000100:00100000:1.0:1761315415.774619:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.774636:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa1b59d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322526976:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315415.774665:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.774709:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526912 00000100:00100000:1.0:1761315415.774736:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526976 00000100:00100000:3.0:1761315415.776825:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4355 00000100:00100000:3.0:1761315415.776834:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322526528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.776847:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4356 00000100:00100000:2.0:1761315415.776851:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cf904e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322526784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.776919:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cf904e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322526784:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (2599us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.776926:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4356 00000100:00100000:2.0:1761315415.776977:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322526784:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.776990:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322526400:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777043:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526464 00000100:00100000:2.0:1761315415.777074:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526592 00000100:00100000:2.0:1761315415.777092:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2729 00000100:00100000:2.0:1761315415.777095:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322526464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777125:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322526464:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (4808us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.777130:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2729 00000100:00100000:2.0:1761315415.777139:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2730 00000100:00100000:2.0:1761315415.777143:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ca269b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322526592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777173:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ca269b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322526592:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (2558us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.777178:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2730 00000100:00100000:2.0:1761315415.777217:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f31517f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322526592:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777284:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777296:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d77d078 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322526848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777330:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526336 00000100:00100000:2.0:1761315415.777378:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620 00000100:00100000:2.0:1761315415.777381:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322526336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777415:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322526336:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (2848us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.777421:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620 00000100:00100000:2.0:1761315415.777460:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322526336:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777493:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527040 00000100:00100000:2.0:1761315415.777515:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 960 00000100:00100000:2.0:1761315415.777518:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322527040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315415.777539:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315415.777570:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322527040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (3207us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.777575:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 960 00000100:00100000:2.0:1761315415.777592:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322526656:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315415.777620:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315415.777696:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315415.777785:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322526528:12345-0@lo:400:kworker/u8:0.0 Request processed in 951us (3834us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315415.777792:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4355 00000100:00100000:3.0:1761315415.777818:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e135009 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322526528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.778541:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322526720:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315415.778569:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.778690:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526656 00000100:00100000:1.0:1761315415.778712:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322526720 00000100:00100000:1.0:1761315415.778726:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1446 00000100:00100000:1.0:1761315415.778733:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322526912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315415.778742:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1447 00000100:00100000:2.0:1761315415.778748:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322526976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315415.778796:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322526912:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (4812us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315415.778803:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322526976:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (4138us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315415.778809:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1446 00000100:00100000:2.0:1761315415.778816:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1447 00000100:00100000:1.0:1761315415.778830:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2138 00000100:00100000:1.0:1761315415.778835:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322526656:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315415.778839:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2139 00000100:00100000:2.0:1761315415.778845:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322526720:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315415.779738:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315415.779742:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315415.779791:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322526656:12345-0@lo:8:kworker/u8:0.0 Request processed in 955us (2170us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315415.779800:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2138 00000100:00100000:1.0:1761315415.779884:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa1b59d6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322526976:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315415.779919:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000006ee7382 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322527040:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315415.779943:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315415.779952:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c03a86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322527104:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315415.779980:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315415.780005:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527104 00000100:00100000:1.0:1761315415.780031:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2731 00000100:00100000:1.0:1761315415.780035:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003741f711 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322527104:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315415.780059:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315415.780092:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003741f711 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322527104:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (112us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315415.780098:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2731 00000100:00100000:1.0:1761315415.780125:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c03a86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322527104:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315415.780147:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322526912:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315415.788577:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315415.788586:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315415.788593:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322526656:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315415.788594:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322526720:12345-0@lo:8:kworker/u8:0.0 Request processed in 9742us (10019us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315415.788608:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2139 00000100:00080000:1.0:1761315415.788610:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315415.788614:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315415.788616:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322526720:0@lo:8:kworker/u8:0.0 00010000:00010000:1.0:1761315416.801316:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315417.805798:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315417.805898:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547520 00000100:00100000:1.0:1761315417.805946:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1621 00000100:00100000:1.0:1761315417.805952:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307547520:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:1.0:1761315417.806022:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307547520:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 69us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315417.806029:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1621 00000100:00100000:3.0:1761315417.808862:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315417.808892:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315417.808947:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547584 00000100:00100000:3.0:1761315417.808961:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315417.808985:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547648 00000100:00100000:3.0:1761315417.808985:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547712 00000100:00100000:2.0:1761315417.809003:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4357 00000100:00100000:2.0:1761315417.809009:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547584:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:3.0:1761315417.809010:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2140 00000100:00100000:3.0:1761315417.809014:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307547712:12345-192.168.203.51@tcp:8:kworker/u8:2.0 00000100:00100000:2.0:1761315417.809064:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307547584:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 55us (208us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315417.809072:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4357 00000100:00100000:2.0:1761315417.809086:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4358 00000100:00100000:2.0:1761315417.809089:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547648:12345-192.168.203.51@tcp:400:kworker/u8:2.0 00000100:00100000:2.0:1761315417.809121:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307547648:12345-192.168.203.51@tcp:400:kworker/u8:2.0 Request processed in 31us (229us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315417.809127:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4358 00010000:02020000:3.0:1761315417.809146:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315417.809172:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307547712:12345-192.168.203.51@tcp:8:kworker/u8:2.0 Request processed in 158us (211us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315417.809491:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2140 00010000:00010000:0.0:1761315417.821663:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315418.333480:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315418.333499:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322527168:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315418.333564:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315418.335912:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527168 00000100:00100000:0.0:1761315418.335999:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2732 00000100:00100000:0.0:1761315418.336005:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322527168:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315418.336045:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315418.336130:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322527168:12345-0@lo:41:osp-pre-1-0.0 Request processed in 124us (2567us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315418.336138:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2732 00000100:00100000:3.0:1761315418.336269:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322527168:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315418.843580:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315419.874530:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315420.891596:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322527232:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315420.891663:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315420.891825:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527232 00000100:00100000:3.0:1761315420.891873:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2733 00000100:00100000:3.0:1761315420.891878:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322527232:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315420.891910:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315420.891947:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322527232:12345-0@lo:41:osp-pre-0-1.0 Request processed in 69us (286us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315420.891953:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2733 00000100:00100000:1.0:1761315420.892032:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322527232:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315420.892512:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322527360:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761315420.892515:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:0.0:1761315420.892520:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315420.892520:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315420.892524:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315420.892531:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3663 00000100:00080000:1.0:1761315420.892534:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315420.892539:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315420.892546:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007cf56814 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322527296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.892547:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315420.892559:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c8817063 x1846869322527616/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315420.892572:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1761315420.892577:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315420.892580:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008850d91b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322527424:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315420.892583:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315420.892586:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315420.892590:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3663 00000100:00080000:1.0:1761315420.892593:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:0.0:1761315420.892597:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315420.892599:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315420.892612:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315420.892614:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a3d67ed x1846869322527680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315420.892625:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.892645:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527360 00000100:00100000:0.0:1761315420.892663:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.892688:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315420.892691:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 961 00000100:00100000:0.0:1761315420.892696:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007520ddc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322527488:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.892698:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322527360:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315420.892722:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315420.892730:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.892750:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527872:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.892767:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322527360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (222us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315420.892774:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 961 00000100:00100000:1.0:1761315420.892785:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e24bfdb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322527808:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315420.892786:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315420.892798:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027585be3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322527744:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315420.892826:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.892837:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527872 00000100:00100000:3.0:1761315420.892858:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.892875:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1448 00000100:00100000:0.0:1761315420.892881:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315420.892883:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527424 00000100:00100000:3.0:1761315420.892892:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322527360:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315420.892926:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527488 00000100:00100000:0.0:1761315420.892928:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527872:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (146us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315420.892936:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1448 00000100:00100000:3.0:1761315420.892936:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527808 00000100:00100000:1.0:1761315420.892956:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527744 00000100:00100000:3.0:1761315420.892962:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4359 00000100:00100000:3.0:1761315420.892968:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322527424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315420.892975:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527872:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.893005:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322527424:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315420.893010:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8817063 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322527616:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315420.893013:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4359 00000100:00100000:2.0:1761315420.893034:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4360 00000100:00100000:0.0:1761315420.893052:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.893078:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527616 00000100:00100000:0.0:1761315420.893095:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2141 00000100:00100000:0.0:1761315420.893098:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322527616:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315420.893202:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315420.893228:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322527616:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (177us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315420.893273:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2141 00000100:00100000:0.0:1761315420.893291:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322527680:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315420.893377:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315420.893399:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527680 00000100:00100000:0.0:1761315420.893420:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2142 00000100:00100000:0.0:1761315420.893424:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c7c91a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322527680:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315420.893505:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315420.893550:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c7c91a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322527680:12345-0@lo:8:kworker/u8:0.0 Request processed in 124us (172us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315420.893556:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2142 00000100:00080000:0.0:1761315420.893682:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315420.893686:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315420.893690:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8817063 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322527616:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315420.893698:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315420.893700:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315420.893702:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322527680:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315420.893734:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527552 00000100:00100000:2.0:1761315420.894006:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322527488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315420.894057:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322527488:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (1328us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315420.894644:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2734 00000100:00100000:0.0:1761315420.894658:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.894675:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4361 00000100:00100000:3.0:1761315420.894682:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322527808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315420.894700:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527552:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (2039us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315420.894709:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2734 00000100:00100000:3.0:1761315420.894727:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322527808:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (1901us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315420.894734:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4361 00000100:00100000:0.0:1761315420.894772:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e24bfdb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322527808:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.894776:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527296 00000100:00100000:3.0:1761315420.894802:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1622 00000100:00100000:3.0:1761315420.894807:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322527296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.894843:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322527296:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (2249us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315420.894852:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007520ddc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322527488:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.894854:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1622 00000100:00100000:0.0:1761315420.894871:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007cf56814 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322527296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315420.894884:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315420.894920:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008850d91b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322527424:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315420.899257:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4362 00000100:00100000:1.0:1761315420.899266:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322527744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315420.899661:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4360 00000100:00100000:1.0:1761315420.899740:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322527744:12345-0@lo:400:kworker/u8:0.0 Request processed in 474us (6884us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315420.899749:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4362 00000100:00100000:3.0:1761315420.901336:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027585be3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322527744:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315421.919235:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315422.923157:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315422.923202:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315422.923243:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315422.923556:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547776 00000100:00100000:2.0:1761315422.923612:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1623 00000100:00100000:2.0:1761315422.923619:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307547776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315422.923683:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307547776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 62us (543us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315422.923690:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1623 00000100:00100000:1.0:1761315422.923773:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547904 00000100:00100000:1.0:1761315422.923805:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547840 00000100:00100000:1.0:1761315422.923819:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4363 00000100:00100000:1.0:1761315422.923823:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307547904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315422.923850:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307547904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (648us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315422.923856:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4363 00000100:00100000:1.0:1761315422.923865:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4364 00000100:00100000:1.0:1761315422.923868:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307547840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315422.923884:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307547840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 16us (642us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315422.923889:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4364 00000100:00100000:2.0:1761315422.926981:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315422.927515:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307547968 00000100:00100000:0.0:1761315422.927567:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2143 00000100:00100000:0.0:1761315422.927572:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547968:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315422.927685:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315422.927757:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307547968:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 183us (782us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315422.927764:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2143 00010000:00010000:3.0:1761315422.940400:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315423.455670:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527936:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315423.455738:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315423.455894:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322527936 00000100:00100000:0.0:1761315423.455945:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2735 00000100:00100000:0.0:1761315423.455951:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527936:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315423.455983:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315423.457456:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322527936:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315423.457567:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322527936:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1613us (1828us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315423.457576:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2735 00010000:00010000:0.0:1761315423.966423:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315424.988657:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315426.011694:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315426.011726:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322528000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315426.011837:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315426.011889:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322528064:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315426.011945:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315426.011953:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315426.011957:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761315426.011962:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315426.011969:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3668 00000100:00080000:2.0:1761315426.011974:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315426.011981:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315426.011995:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322528320:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315426.012012:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fb9fba07 x1846869322528384/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315426.012029:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315426.012031:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315426.012036:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315426.012037:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315426.012043:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3668 00000100:00080000:2.0:1761315426.012045:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315426.012049:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315426.012054:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315426.012063:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008c88b345 x1846869322528448/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315426.012065:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322528256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012107:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315426.012143:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315426.012154:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322528192:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012157:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322528640:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012214:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315426.012321:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322528128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012510:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322528704:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315426.012543:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315426.012601:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322528384:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012610:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528064 00000100:00100000:0.0:1761315426.012657:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315426.012691:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1624 00000100:00100000:0.0:1761315426.012697:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c88b345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322528448:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012703:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322528064:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315426.012723:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315426.012772:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528320 00000100:00100000:1.0:1761315426.012776:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322528064:12345-0@lo:400:kworker/u8:0.0 Request processed in 73us (816us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315426.012785:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1624 00000100:00100000:0.0:1761315426.012822:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528704 00000100:00100000:1.0:1761315426.012837:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528640 00000100:00100000:0.0:1761315426.012856:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2736 00000100:00100000:0.0:1761315426.012866:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322528320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.012881:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528384 00000100:00100000:1.0:1761315426.012897:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528448 00000100:00100000:1.0:1761315426.012917:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1449 00000100:00100000:1.0:1761315426.012923:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a748b8e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322528640:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315426.012937:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322528320:12345-0@lo:400:kworker/u8:0.0 Request processed in 71us (909us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315426.012948:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2736 00000100:00100000:1.0:1761315426.012973:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a748b8e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322528640:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (758us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315426.012981:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2737 00000100:00100000:1.0:1761315426.012987:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1449 00000100:00100000:0.0:1761315426.012996:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322528704:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315426.013013:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2144 00000100:00100000:1.0:1761315426.013018:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322528384:12345-0@lo:8:kworker/u8:0.0 00000004:00000024:0.0:1761315426.013030:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315426.013070:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322528704:12345-0@lo:41:osp-pre-0-1.0 Request processed in 77us (526us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315426.013076:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2737 00000100:00100000:0.0:1761315426.013102:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528256 00010000:02020000:1.0:1761315426.013134:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315426.013149:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4365 00000100:00100000:0.0:1761315426.013156:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322528256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.013187:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322528384:12345-0@lo:8:kworker/u8:0.0 Request processed in 166us (531us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315426.013195:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2144 00000100:00100000:1.0:1761315426.013216:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2145 00000100:00100000:0.0:1761315426.013223:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322528256:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (1111us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315426.013228:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322528448:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315426.013247:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4365 00010000:02020000:1.0:1761315426.013312:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315426.013341:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322528448:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (620us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315426.013375:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2145 00000100:00100000:2.0:1761315426.013463:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315426.013796:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315426.013820:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322528512:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315426.013845:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315426.013882:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528000 00000100:00100000:3.0:1761315426.013925:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 962 00000100:00100000:3.0:1761315426.013929:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322528000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315426.013951:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315426.013979:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322528000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (2146us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315426.013984:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 962 00000100:00100000:3.0:1761315426.014003:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528128 00000100:00100000:1.0:1761315426.014542:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528192 00000100:00100000:2.0:1761315426.014570:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322528576:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.014592:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2738 00000100:00100000:1.0:1761315426.014599:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322528192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315426.014619:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315426.014654:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322528192:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (1192us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315426.014662:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2738 00000100:00100000:2.0:1761315426.014669:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322528192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315426.014712:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528512 00000100:00100000:1.0:1761315426.014740:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322528704:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315426.014755:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322528640:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.014766:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322528256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.014777:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322528320:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.014786:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322528064:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315426.014801:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322528000:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315426.015205:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528576 00000100:00100000:0.0:1761315426.015233:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4366 00000100:00100000:0.0:1761315426.015238:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322528128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315426.015264:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4367 00000100:00100000:2.0:1761315426.015272:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bdd257c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322528576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315426.015288:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322528128:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (1492us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315426.015297:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4366 00000100:00100000:0.0:1761315426.015315:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4368 00000100:00100000:0.0:1761315426.015322:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322528512:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315426.015379:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bdd257c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322528576:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (725us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315426.015390:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4367 00000100:00100000:0.0:1761315426.015413:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322528512:12345-0@lo:400:kworker/u8:0.0 Request processed in 87us (1563us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315426.015426:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4368 00000100:00100000:2.0:1761315426.015464:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322528576:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315426.015500:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315426.015531:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322528128:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315426.015550:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322528512:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315426.016410:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315426.016418:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315426.016424:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322528384:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315426.016450:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315426.016453:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315426.016455:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c88b345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322528448:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315427.036795:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315428.040087:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315428.040162:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315428.040204:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315428.040312:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548096 00000100:00100000:3.0:1761315428.040426:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548160 00000100:00100000:3.0:1761315428.040452:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4369 00000100:00100000:3.0:1761315428.040461:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307548096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315428.040574:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4370 00000100:00100000:0.0:1761315428.040579:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307548160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315428.040668:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307548160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (465us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315428.040677:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4370 00000100:00100000:0.0:1761315428.040835:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307548096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 374us (760us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315428.040842:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4369 00000100:00100000:1.0:1761315428.041110:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548032 00000100:00100000:1.0:1761315428.041153:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1625 00000100:00100000:1.0:1761315428.041158:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307548032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315428.041199:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307548032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 40us (1037us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315428.041206:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1625 00000100:00100000:0.0:1761315428.042952:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315428.043058:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548224 00000100:00100000:2.0:1761315428.043279:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2146 00000100:00100000:2.0:1761315428.043286:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548224:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315428.043451:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315428.044146:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548224:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 858us (1199us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315428.044157:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2146 00010000:00010000:0.0:1761315428.059627:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315428.572249:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315428.572279:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322528768:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315428.572388:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315428.572623:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528768 00000100:00100000:1.0:1761315428.572688:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2739 00000100:00100000:1.0:1761315428.572696:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322528768:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315428.572742:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315428.572803:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322528768:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315428.572812:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2739 00000100:00100000:2.0:1761315428.573095:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322528768:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315429.083522:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315430.108227:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315431.131525:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315431.131694:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315431.131714:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322528832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315431.131777:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.131850:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528832 00000100:00100000:2.0:1761315431.131930:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 963 00000100:00100000:2.0:1761315431.131938:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322528832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315431.131980:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315431.132064:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322528832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (286us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315431.132077:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 963 00000100:00080000:3.0:1761315431.132078:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315431.132093:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315431.132097:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315431.132103:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3673 00000100:00080000:3.0:1761315431.132107:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315431.132113:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315431.132133:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322529088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132134:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008e76633c x1846869322529280/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315431.132158:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315431.132161:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315431.132163:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315431.132167:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3673 00000100:00080000:3.0:1761315431.132170:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315431.132175:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315431.132190:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004f9b8f16 x1846869322529344/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315431.132197:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.132241:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322529472:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132272:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529088 00000100:00100000:2.0:1761315431.132278:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.132306:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315431.132317:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322529024:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132323:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2740 00000100:00100000:3.0:1761315431.132333:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322529088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.132386:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.132408:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322529408:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132430:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322529088:12345-0@lo:400:kworker/u8:2.0 Request processed in 94us (242us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315431.132440:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2740 00000100:00100000:2.0:1761315431.132447:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315431.132490:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529472 00000100:00100000:2.0:1761315431.132494:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322529088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132532:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529024 00000100:00100000:3.0:1761315431.132568:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529408 00000100:00100000:3.0:1761315431.132589:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4371 00000100:00100000:3.0:1761315431.132592:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322529472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315431.132640:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322529472:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315431.132647:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4371 00000100:00100000:2.0:1761315431.132728:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4372 00000100:00100000:2.0:1761315431.132739:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322529024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315431.132754:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322528896:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315431.132797:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322529024:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (412us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315431.132812:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.132820:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4372 00000100:00100000:1.0:1761315431.132844:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322529152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.132852:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4373 00000100:00100000:2.0:1761315431.132860:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322529408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315431.132878:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315431.132899:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322529536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.132909:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322529408:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (463us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315431.132918:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4373 00000100:00100000:1.0:1761315431.132934:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.132939:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529152 00000100:00100000:1.0:1761315431.132966:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315431.132974:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4374 00000100:00100000:1.0:1761315431.132978:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322528960:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.132982:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1b735c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322529152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315431.133017:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.133043:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1b735c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322529152:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315431.133048:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322529216:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.133057:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4374 00000100:00100000:1.0:1761315431.133098:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.133117:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322529024:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.133142:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322529408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315431.133150:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322529152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315431.133159:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322529472:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315431.133201:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008e76633c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322529280:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315431.133229:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315431.133257:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322529344:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315431.133260:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529536 00000100:00100000:1.0:1761315431.133290:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315431.133315:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529280 00000100:00100000:2.0:1761315431.133370:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529344 00000100:00100000:2.0:1761315431.133399:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1450 00000100:00100000:2.0:1761315431.133404:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322529536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315431.133418:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322528832:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315431.133459:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322529536:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (527us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315431.133468:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1450 00000100:00100000:0.0:1761315431.133477:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2147 00000100:00100000:0.0:1761315431.133485:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322529280:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315431.133492:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2148 00000100:00100000:2.0:1761315431.133497:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322529344:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315431.133692:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315431.133731:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322529280:12345-0@lo:8:kworker/u8:2.0 Request processed in 245us (501us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315431.133738:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2147 00000100:00100000:0.0:1761315431.133793:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322529536:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315431.133826:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528960 00000100:00100000:0.0:1761315431.133853:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1626 00000100:00100000:0.0:1761315431.133857:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322528960:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:2.0:1761315431.133871:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315431.133895:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322528960:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (877us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315431.133903:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1626 00000100:00100000:2.0:1761315431.133925:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322529344:12345-0@lo:8:kworker/u8:2.0 Request processed in 424us (633us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315431.133942:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2148 00000100:00100000:0.0:1761315431.133945:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322528896 00000100:00100000:0.0:1761315431.133975:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529216 00000100:00100000:2.0:1761315431.133992:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2741 00000100:00100000:2.0:1761315431.134002:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322528896:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315431.134012:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2742 00000100:00100000:0.0:1761315431.134019:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e0ab9f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322529216:12345-0@lo:400:kworker/u8:2.0 00000004:00000024:2.0:1761315431.134042:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315431.134064:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e0ab9f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322529216:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (982us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315431.134073:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2742 00000100:00100000:2.0:1761315431.134100:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322528896:12345-0@lo:41:osp-pre-0-1.0 Request processed in 98us (1297us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315431.134108:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2741 00000100:00100000:0.0:1761315431.134131:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322528960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315431.134144:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322529216:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315431.134165:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322528896:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761315431.134682:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315431.134687:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315431.134692:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322529344:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315431.134714:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315431.134717:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315431.134720:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008e76633c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322529280:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315432.155533:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315433.161463:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315433.163954:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315433.164385:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548480 00000100:00100000:3.0:1761315433.164471:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2149 00000100:00100000:3.0:1761315433.164479:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548480:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315433.164699:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315433.164845:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548480:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 364us (3392us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315433.164855:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2149 00000100:00100000:0.0:1761315433.165336:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315433.165387:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315433.165394:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548288 00000100:00100000:1.0:1761315433.165519:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1627 00000100:00100000:1.0:1761315433.165525:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307548288:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315433.165578:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307548288:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 52us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315433.165586:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1627 00000100:00100000:2.0:1761315433.165704:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548352 00000100:00100000:2.0:1761315433.165756:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4375 00000100:00100000:2.0:1761315433.165762:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006ee7382 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307548352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315433.165813:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006ee7382 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307548352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 50us (1868us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315433.165820:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4375 00000100:00100000:1.0:1761315433.166336:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548416 00000100:00100000:1.0:1761315433.166394:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4376 00000100:00100000:1.0:1761315433.166398:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307548416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315433.166445:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307548416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 46us (1060us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315433.166452:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4376 00010000:00010000:1.0:1761315433.180236:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315433.691649:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322529600:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315433.691750:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315433.691913:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529600 00000100:00100000:3.0:1761315433.691988:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2743 00000100:00100000:3.0:1761315433.691997:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007cf56814 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322529600:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315433.692055:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315433.692135:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007cf56814 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322529600:12345-0@lo:41:osp-pre-1-0.0 Request processed in 138us (391us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315433.692143:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2743 00000100:00100000:1.0:1761315433.694102:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322529600:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315434.203619:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315435.227569:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315436.251543:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322529664:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315436.251552:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315436.251558:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315436.251562:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315436.251567:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3678 00000100:00080000:2.0:1761315436.251570:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315436.251575:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315436.251602:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fb9fba07 x1846869322530112/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315436.251605:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322529856:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315436.251609:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315436.251627:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315436.251627:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315436.251633:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315436.251635:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761315436.251638:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f86d06ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322529792:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315436.251641:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3678 00000100:00080000:2.0:1761315436.251645:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315436.251650:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315436.251651:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315436.251668:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eaa26ccb x1846869322530176/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315436.251669:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322529920:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315436.251682:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315436.251709:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322530112:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315436.251711:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007520ddc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322530048:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315436.251716:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315436.251760:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315436.251762:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322530304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315436.251763:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315436.251772:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529792 00000100:00100000:1.0:1761315436.251785:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322530176:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315436.251801:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322530432:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.251806:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315436.251817:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315436.251821:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529920 00000100:00100000:2.0:1761315436.251839:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530304 00000100:00100000:2.0:1761315436.251854:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4377 00000100:00100000:2.0:1761315436.251859:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322529792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315436.253124:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529856 00000100:00100000:1.0:1761315436.253168:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2744 00000100:00100000:1.0:1761315436.253173:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e24bfdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322529856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253279:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322529792:12345-0@lo:400:kworker/u8:0.0 Request processed in 1417us (1598us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.253288:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4377 00000100:00100000:2.0:1761315436.253308:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4378 00000100:00100000:2.0:1761315436.253311:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322529920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253367:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322529920:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (1657us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.253373:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4378 00000100:00100000:2.0:1761315436.253385:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4379 00000100:00100000:2.0:1761315436.253389:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322530304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253415:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322530304:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (1613us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.253421:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4379 00000100:00100000:0.0:1761315436.253443:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315436.253471:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322530240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315436.253482:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f86d06ac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322529792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253505:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315436.253522:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530048 00000100:00100000:2.0:1761315436.253545:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530240 00000100:00100000:0.0:1761315436.253558:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 964 00000100:00100000:3.0:1761315436.253560:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322530304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315436.253567:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322530048:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315436.253579:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4380 00000100:00100000:3.0:1761315436.253580:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322529920:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253587:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322530240:12345-0@lo:400:kworker/u8:0.0 00002000:00000024:0.0:1761315436.253592:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315436.253605:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:3.0:1761315436.253613:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322529728:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315436.253631:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322530240:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (125us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.253638:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4380 00000100:00100000:0.0:1761315436.253647:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322530048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (1888us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315436.253655:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315436.253661:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 964 00000100:00100000:2.0:1761315436.253674:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322530240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.253683:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322529984:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315436.253699:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529728 00000100:00100000:3.0:1761315436.253711:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315436.253730:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529664 00000100:00100000:2.0:1761315436.253757:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1628 00000100:00100000:2.0:1761315436.253760:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322529664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.253796:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322529664:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (2173us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.253802:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1628 00000100:00100000:1.0:1761315436.255612:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e24bfdb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322529856:12345-0@lo:400:kworker/u8:0.0 Request processed in 2431us (3958us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315436.255626:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2744 00000100:00100000:2.0:1761315436.256708:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322529984 00000100:00100000:2.0:1761315436.256754:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2745 00000100:00100000:3.0:1761315436.256758:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530368:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.256762:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322529728:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315436.256800:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315436.256814:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315436.256851:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322529664:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.256878:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322529728:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315436.256934:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530112 00000100:00100000:2.0:1761315436.256942:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322529728:12345-0@lo:41:osp-pre-0-1.0 Request processed in 180us (3292us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.256951:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2745 00000100:00100000:3.0:1761315436.256963:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530176 00000100:00100000:2.0:1761315436.256972:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2746 00000100:00100000:2.0:1761315436.256978:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322529984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.256984:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530432 00000100:00100000:3.0:1761315436.257004:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530368 00000100:00100000:3.0:1761315436.257020:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1451 00000100:00100000:2.0:1761315436.257023:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322529984:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (3313us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315436.257028:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322530432:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315436.257034:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2746 00000100:00100000:2.0:1761315436.257074:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1452 00000100:00100000:3.0:1761315436.257080:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322530432:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (3638us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.257081:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322530368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.257093:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1451 00000100:00100000:3.0:1761315436.257109:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2150 00000100:00100000:3.0:1761315436.257113:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322530112:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315436.257123:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322530368:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (309us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315436.257129:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1452 00000100:00100000:2.0:1761315436.257144:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2151 00000100:00100000:2.0:1761315436.257147:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8817063 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322530176:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315436.257208:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315436.257246:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322530112:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (5488us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315436.257252:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2150 00000100:00100000:3.0:1761315436.257287:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322529984:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.257297:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530368:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315436.257331:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315436.257336:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315436.257339:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322530112:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315436.257417:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322529856:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.257467:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322530432:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315436.257480:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007520ddc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322530048:0@lo:13:osp-pre-1-1.0 00010000:02020000:2.0:1761315436.257687:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315436.257718:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8817063 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322530176:12345-0@lo:8:kworker/u8:0.0 Request processed in 570us (5903us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315436.257725:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2151 00000100:00080000:3.0:1761315436.257955:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315436.257959:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315436.257963:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322530176:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315437.275642:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315438.281276:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315438.281328:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315438.281390:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315438.281429:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315438.282578:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548672 00000100:00100000:2.0:1761315438.282646:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548608 00000100:00100000:2.0:1761315438.282671:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4381 00000100:00100000:2.0:1761315438.282677:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307548672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315438.282747:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307548672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (1419us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315438.282753:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4381 00000100:00100000:2.0:1761315438.282769:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4382 00000100:00100000:2.0:1761315438.282773:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307548608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315438.282795:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307548608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (1405us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315438.282800:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4382 00000100:00100000:2.0:1761315438.282820:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548736 00000100:00100000:2.0:1761315438.282837:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2152 00000100:00100000:2.0:1761315438.282841:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ddb024e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548736:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315438.282927:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315438.282954:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ddb024e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548736:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 112us (1525us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315438.282960:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2152 00000100:00100000:3.0:1761315438.283669:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548544 00000100:00100000:3.0:1761315438.283712:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1629 00000100:00100000:3.0:1761315438.283717:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307548544:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315438.283754:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307548544:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 37us (2488us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315438.283761:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1629 00010000:00010000:0.0:1761315438.299705:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315438.811618:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055861db2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322530496:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315438.811688:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315438.811800:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530496 00000100:00100000:0.0:1761315438.812208:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2747 00000100:00100000:0.0:1761315438.812215:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322530496:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315438.812248:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315438.812373:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322530496:12345-0@lo:41:osp-pre-1-0.0 Request processed in 157us (685us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315438.812380:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2747 00000100:00100000:3.0:1761315438.812460:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055861db2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322530496:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315439.323513:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315440.347524:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315441.372534:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315441.372614:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.372659:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315441.372667:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530624:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315441.372671:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315441.372695:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315441.372777:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530560 00000100:00100000:1.0:1761315441.372829:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 965 00000100:00100000:1.0:1761315441.372835:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9d25e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322530560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315441.372864:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315441.372904:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9d25e52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322530560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.372910:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 965 00000100:00100000:3.0:1761315441.373267:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530560:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315441.373335:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530624 00000100:00100000:0.0:1761315441.373385:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2748 00000100:00100000:0.0:1761315441.373390:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322530624:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315441.373410:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315441.373449:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322530624:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (755us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315441.373455:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2748 00000100:00100000:3.0:1761315441.373677:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322530624:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315441.374516:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315441.375130:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322530688:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315441.375184:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.375302:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530688 00000100:00100000:3.0:1761315441.375368:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1630 00000100:00100000:3.0:1761315441.375373:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322530688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315441.375416:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322530688:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (233us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315441.375423:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1630 00000100:00100000:2.0:1761315441.375530:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322530688:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315441.375627:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315441.375631:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315441.375634:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315441.375639:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3683 00000100:00080000:0.0:1761315441.375641:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315441.375644:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315441.375659:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008b165e1a x1846869322530944/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315441.375671:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315441.375674:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315441.375675:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315441.375677:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3683 00000100:00080000:0.0:1761315441.375678:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315441.375680:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315441.375686:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000049738da9 x1846869322531008/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315441.375715:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a30fa38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322530880:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315441.375739:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315441.375751:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315441.375755:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322530816:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315441.375773:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315441.375781:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322531200:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315441.375797:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315441.375816:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531200 00000100:00100000:0.0:1761315441.375838:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1453 00000100:00100000:0.0:1761315441.375841:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d52d58b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322531200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315441.375866:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d52d58b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322531200:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (69us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315441.375874:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1453 00000100:00100000:0.0:1761315441.375895:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1424fc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322531200:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315441.375948:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008b165e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322530944:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315441.375968:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.375985:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049738da9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322531008:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315441.375999:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.376029:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322531072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315441.376046:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.376060:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315441.376063:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322530752:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315441.376077:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315441.376086:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055861db2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322531136:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315441.376098:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315441.376256:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530816 00000100:00100000:1.0:1761315441.376279:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531072 00000100:00100000:1.0:1761315441.376290:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530752 00000100:00100000:1.0:1761315441.376297:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531136 00000100:00100000:1.0:1761315441.376306:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4383 00000100:00100000:1.0:1761315441.376309:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2a5a930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322530816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376370:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2a5a930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322530816:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (596us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.376377:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4383 00000100:00100000:1.0:1761315441.376388:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4384 00000100:00100000:1.0:1761315441.376391:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322531072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376420:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322531072:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (373us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.376425:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4384 00000100:00100000:1.0:1761315441.376434:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4385 00000100:00100000:1.0:1761315441.376436:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322530752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376454:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322530752:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (378us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.376459:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4385 00000100:00100000:1.0:1761315441.376465:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4386 00000100:00100000:1.0:1761315441.376468:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322531136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376483:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322531136:12345-0@lo:400:kworker/u8:0.0 Request processed in 14us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.376487:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4386 00000100:00100000:1.0:1761315441.376507:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530880 00000100:00100000:1.0:1761315441.376523:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2749 00000100:00100000:1.0:1761315441.376526:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cccdaf22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322530880:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376546:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cccdaf22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322530880:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (808us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315441.376551:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2749 00000100:00100000:1.0:1761315441.376581:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322530944 00000100:00100000:1.0:1761315441.376592:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531008 00000100:00100000:1.0:1761315441.376602:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2153 00000100:00100000:1.0:1761315441.376606:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322530944:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315441.376684:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315441.376708:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322530944:12345-0@lo:8:kworker/u8:0.0 Request processed in 102us (740us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315441.376714:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2153 00000100:00100000:1.0:1761315441.376721:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2154 00000100:00100000:1.0:1761315441.376724:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322531008:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315441.376790:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315441.376806:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322531008:12345-0@lo:8:kworker/u8:0.0 Request processed in 81us (807us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315441.376810:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2154 00000100:00100000:1.0:1761315441.376846:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013111502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322530752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376854:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055861db2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322531136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315441.376860:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322531072:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315441.376892:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315441.376896:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315441.376898:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008b165e1a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322530944:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315441.376904:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315441.376906:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315441.376907:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049738da9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322531008:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315441.377433:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045e42374 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322530816:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315441.377444:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a30fa38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322530880:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315442.396261:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315443.402051:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315443.402859:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315443.402899:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315443.403598:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548800 00000100:00100000:2.0:1761315443.403649:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1631 00000100:00100000:2.0:1761315443.403657:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307548800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315443.403727:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307548800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (1686us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315443.403734:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1631 00000100:00100000:1.0:1761315443.406530:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315443.406581:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548992 00000100:00100000:1.0:1761315443.406618:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2155 00000100:00100000:1.0:1761315443.406624:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dd56348 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548992:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761315443.406719:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548928 00010000:02020000:1.0:1761315443.406728:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315443.406755:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307548864 00000100:00100000:3.0:1761315443.406777:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4387 00000100:00100000:3.0:1761315443.406782:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307548928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315443.406800:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dd56348 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307548992:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 173us (275us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315443.406810:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2155 00000100:00100000:3.0:1761315443.406827:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307548928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 43us (3969us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315443.406835:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4387 00000100:00100000:3.0:1761315443.406852:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4388 00000100:00100000:3.0:1761315443.406856:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307548864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315443.406883:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307548864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (3984us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315443.406890:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4388 00010000:00010000:0.0:1761315443.421226:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315443.931639:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315443.931662:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004d72fb3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322531264:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315443.931729:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315443.931829:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531264 00000100:00100000:1.0:1761315443.932455:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2750 00000100:00100000:1.0:1761315443.932463:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007520ddc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322531264:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315443.932498:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315443.932567:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007520ddc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322531264:12345-0@lo:41:osp-pre-1-0.0 Request processed in 103us (838us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315443.932575:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2750 00000100:00100000:3.0:1761315443.932691:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004d72fb3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322531264:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315444.443512:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315445.467519:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315446.491514:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315446.491752:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315446.491758:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315446.491761:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315446.491770:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3688 00000100:00080000:1.0:1761315446.491773:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:0.0:1761315446.491776:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003741f711 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322531456:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315446.491781:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315446.491802:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a10b3b20 x1846869322531648/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315446.491820:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315446.491823:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315446.491824:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315446.491828:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3688 00000100:00080000:1.0:1761315446.491830:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315446.491834:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315446.491843:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315446.491846:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001345af7d x1846869322531712/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315446.491887:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322531840:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315446.491898:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322531648:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315446.491914:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315446.491932:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315446.491937:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003952b204 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322531520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315446.491943:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315446.492024:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ca269b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322531392:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315446.492063:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060e3fc1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322531328:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.492170:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315446.492202:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322531904:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.492361:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315446.492363:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315446.492373:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315446.492392:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000059c6fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322531584:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.492399:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531456 00000100:00100000:2.0:1761315446.492403:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322531776:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315446.492428:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315446.492440:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315446.492449:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322531968:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315446.492476:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315446.492490:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531840 00000100:00100000:2.0:1761315446.492523:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531520 00000100:00100000:2.0:1761315446.492551:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531392 00000100:00100000:2.0:1761315446.492568:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531776 00000100:00100000:2.0:1761315446.492586:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4389 00000100:00100000:2.0:1761315446.492592:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009716890d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322531840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.492865:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009716890d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322531840:12345-0@lo:400:kworker/u8:2.0 Request processed in 272us (951us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.492872:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4389 00000100:00100000:2.0:1761315446.492889:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4390 00000100:00100000:2.0:1761315446.492892:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005819e23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322531520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.492914:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005819e23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322531520:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (746us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.492919:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4390 00000100:00100000:2.0:1761315446.492928:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4391 00000100:00100000:2.0:1761315446.492931:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322531392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.492948:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322531392:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (608us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.492953:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4391 00000100:00100000:2.0:1761315446.492961:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4392 00000100:00100000:2.0:1761315446.492964:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322531776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493000:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531584 00000100:00100000:2.0:1761315446.493000:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322531776:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (562us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.493007:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4392 00000100:00100000:0.0:1761315446.493035:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2751 00000100:00100000:0.0:1761315446.493041:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbec2d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322531456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493047:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322532032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315446.493080:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315446.493089:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbec2d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322531456:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (1248us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315446.493097:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2751 00000100:00100000:0.0:1761315446.493111:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2752 00000100:00100000:2.0:1761315446.493115:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ca269b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322531392:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493117:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322531584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493133:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322531776:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493164:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532032 00000100:00100000:0.0:1761315446.493182:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322531584:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (751us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315446.493190:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2752 00000100:00100000:2.0:1761315446.493194:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 966 00000100:00100000:2.0:1761315446.493199:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322532032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315446.493222:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315446.493241:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315446.493250:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7076cb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322532096:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315446.493260:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322532032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.493267:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 966 00000100:00100000:0.0:1761315446.493280:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315446.493300:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322532032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315446.493326:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532096 00000100:00100000:2.0:1761315446.493331:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003952b204 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322531520:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493366:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322531840:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493372:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2753 00000100:00100000:0.0:1761315446.493378:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000498385af pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322532096:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315446.493380:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003741f711 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322531456:0@lo:400:kworker/u8:2.0 00000004:00000024:0.0:1761315446.493400:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315446.493434:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000498385af pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322532096:12345-0@lo:41:osp-pre-0-1.0 Request processed in 57us (156us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315446.493440:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2753 00000100:00100000:2.0:1761315446.493442:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531328 00000100:00100000:2.0:1761315446.493468:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632 00000100:00100000:2.0:1761315446.493473:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322531328:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493484:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7076cb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322532096:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315446.493505:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322531328:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (1166us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315446.493512:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632 00000100:00100000:0.0:1761315446.493533:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531648 00000100:00100000:0.0:1761315446.493556:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531904 00000100:00100000:2.0:1761315446.493563:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060e3fc1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322531328:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493576:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531968 00000100:00100000:2.0:1761315446.493577:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000059c6fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322531584:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493603:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1454 00000100:00100000:0.0:1761315446.493609:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322531904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493615:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1455 00000100:00100000:2.0:1761315446.493621:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322531968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315446.493647:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322531904:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (1304us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315446.493654:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1454 00000100:00100000:2.0:1761315446.493670:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322531968:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (1192us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315446.493672:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2156 00000100:00100000:0.0:1761315446.493679:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322531648:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493682:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1455 00000100:00100000:2.0:1761315446.493710:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322531968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315446.493772:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322531904:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315446.493786:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315446.493820:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322531648:12345-0@lo:8:kworker/u8:2.0 Request processed in 141us (1881us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315446.493827:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2156 00000100:00100000:1.0:1761315446.493849:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001345af7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322531712:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315446.493890:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315446.493925:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315446.493929:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315446.493932:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322531648:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315446.494170:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322531712 00000100:00100000:0.0:1761315446.494246:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2157 00000100:00100000:0.0:1761315446.494250:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322531712:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315446.494324:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315446.495021:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322531712:12345-0@lo:8:kworker/u8:2.0 Request processed in 769us (1129us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315446.495028:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2157 00000100:00080000:1.0:1761315446.495066:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315446.495071:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315446.495075:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001345af7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322531712:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315447.515559:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315448.521577:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315448.521623:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315448.521652:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549248 00000100:00100000:1.0:1761315448.521663:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315448.521694:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2158 00000100:00100000:0.0:1761315448.521702:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307549248:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761315448.521711:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315448.521735:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549056 00000100:00100000:1.0:1761315448.521761:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549120 00000100:00100000:2.0:1761315448.521773:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1633 00000100:00100000:2.0:1761315448.521780:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307549056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00010000:02020000:0.0:1761315448.521796:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315448.521796:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549184 00000100:00100000:1.0:1761315448.521823:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4393 00000100:00100000:1.0:1761315448.521834:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307549120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315448.521847:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307549056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (182us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315448.521848:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307549248:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 143us (275us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315448.521862:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1633 00000100:00100000:0.0:1761315448.521863:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2158 00000100:00100000:1.0:1761315448.521878:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307549120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 48us (254us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315448.521886:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4393 00000100:00100000:1.0:1761315448.521907:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4394 00000100:00100000:1.0:1761315448.521912:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307549184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315448.521943:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307549184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 30us (236us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315448.523004:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4394 00010000:00010000:0.0:1761315448.540316:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315449.052132:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315449.052155:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7076cb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322532160:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315449.052222:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315449.052327:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532160 00000100:00100000:0.0:1761315449.052406:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2754 00000100:00100000:0.0:1761315449.052413:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322532160:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315449.052445:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315449.052496:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322532160:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315449.052504:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2754 00000100:00100000:3.0:1761315449.052648:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7076cb1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322532160:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315449.563781:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315450.587902:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315451.611507:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322532224:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315451.611507:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf5b9d47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322532288:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.611508:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315451.611522:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322532416:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315451.611537:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315451.611543:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315451.611546:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315451.611553:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3694 00000100:00080000:1.0:1761315451.611556:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315451.611561:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315451.611581:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000099ff5107 x1846869322532672/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315451.611596:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315451.611597:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315451.611597:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315451.611605:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315451.611610:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315451.611612:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315451.611616:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3694 00000100:00080000:1.0:1761315451.611618:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315451.611621:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315451.611634:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000465cf088 x1846869322532736/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315451.611638:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322532352:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315451.611640:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322532480:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.611641:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322532544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.611682:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315451.611692:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315451.611695:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315451.611703:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099ff5107 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322532672:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315451.611712:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aedb9ccf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322532800:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315451.611727:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e88e689 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322532864:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.611729:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030335973 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322532928:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.611748:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315451.611752:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315451.611777:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000465cf088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322532736:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315451.611779:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315451.611781:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315451.611781:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315451.611790:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002bbf3014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322532608:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.611824:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315451.611829:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315451.611838:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532416 00000100:00100000:0.0:1761315451.611845:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532224 00000100:00100000:3.0:1761315451.611855:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1b80cdb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322532992:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.611869:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532352 00000100:00100000:2.0:1761315451.611886:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532544 00000100:00100000:0.0:1761315451.611887:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532480 00000100:00100000:3.0:1761315451.611887:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315451.611910:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 967 00000100:00100000:0.0:1761315451.611913:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532608 00000100:00100000:1.0:1761315451.611919:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322532352:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315451.611922:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532800 00000100:00100000:3.0:1761315451.611934:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532864 00000100:00100000:2.0:1761315451.611960:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4395 00002000:00000024:1.0:1761315451.611963:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315451.611970:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2755 00000100:00100000:2.0:1761315451.611970:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322532416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.611976:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4396 00000100:00100000:0.0:1761315451.611979:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322532224:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315451.611985:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322532544:12345-0@lo:400:kworker/u8:0.0 00000004:00000024:0.0:1761315451.612008:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315451.612013:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322532352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 93us (331us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315451.612022:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 967 00000100:00100000:2.0:1761315451.612026:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322532416:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (435us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315451.612034:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4395 00000100:00100000:3.0:1761315451.612039:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322532544:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (348us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315451.612048:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4396 00000100:00100000:1.0:1761315451.612053:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532672 00000100:00100000:2.0:1761315451.612058:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4397 00000100:00100000:2.0:1761315451.612064:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000306fc987 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322532800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.612068:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4398 00000100:00100000:1.0:1761315451.612076:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2159 00000100:00100000:3.0:1761315451.612077:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d72fb3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322532864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.612084:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322532672:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315451.612127:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d55f4c70 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322532224:12345-0@lo:41:osp-pre-0-1.0 Request processed in 149us (539us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315451.612131:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000306fc987 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322532800:12345-0@lo:400:kworker/u8:0.0 Request processed in 64us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315451.612132:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d72fb3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322532864:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315451.612142:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2755 00000100:00100000:2.0:1761315451.612147:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4397 00000100:00100000:3.0:1761315451.612154:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4398 00000100:00100000:0.0:1761315451.612163:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2756 00000100:00100000:0.0:1761315451.612169:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7076cb1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322532480:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315451.612174:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315451.612194:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532928 00000100:00100000:0.0:1761315451.612211:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7076cb1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322532480:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (524us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315451.612215:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aedb9ccf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322532800:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.612223:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532736 00000100:00100000:0.0:1761315451.612228:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2756 00000100:00100000:1.0:1761315451.612234:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322532672:12345-0@lo:8:kworker/u8:0.0 Request processed in 149us (487us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315451.612239:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322532416:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.612240:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532992 00000100:00100000:1.0:1761315451.612245:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2159 00000100:00100000:0.0:1761315451.612248:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2757 00000100:00100000:0.0:1761315451.612254:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fc39ebc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322532608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.612262:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322532352:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315451.612268:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1456 00000100:00100000:1.0:1761315451.612274:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322532928:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315451.612289:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fc39ebc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322532608:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (461us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315451.612294:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1457 00000100:00100000:0.0:1761315451.612298:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2757 00000100:00100000:2.0:1761315451.612304:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc7bc95e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322532992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315451.612304:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002bbf3014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322532608:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.612316:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322532928:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (541us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315451.612324:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322532288 00000100:00100000:1.0:1761315451.612324:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1456 00000100:00100000:1.0:1761315451.612372:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2160 00000100:00100000:0.0:1761315451.612377:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1634 00000100:00100000:1.0:1761315451.612381:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322532736:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315451.612387:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322532288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.612387:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc7bc95e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322532992:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (499us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315451.612401:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030335973 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322532928:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315451.612402:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1457 00000100:00100000:3.0:1761315451.612420:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322532544:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315451.612429:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322532288:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (840us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315451.612435:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf5b9d47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322532288:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315451.612437:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1634 00010000:02020000:1.0:1761315451.612465:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315451.612466:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1b80cdb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322532992:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.612502:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322532736:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (679us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315451.612511:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2160 00010000:00010000:2.0:1761315451.612532:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315451.612539:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e88e689 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322532864:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315451.612549:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315451.612555:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315451.612559:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322532480:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315451.612562:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099ff5107 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322532672:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315451.612581:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315451.612585:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315451.612588:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322532224:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315451.612591:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000465cf088 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322532736:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315452.635530:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315453.640097:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315453.641740:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549312 00000100:00100000:2.0:1761315453.641801:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1635 00000100:00100000:2.0:1761315453.641807:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307549312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315453.641885:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307549312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (1794us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315453.641892:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1635 00000100:00100000:3.0:1761315453.645333:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315453.645626:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549504 00000100:00100000:2.0:1761315453.646023:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2161 00000100:00100000:2.0:1761315453.646034:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000026030406 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307549504:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315453.646131:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315453.646191:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315453.646246:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549440 00000100:00100000:1.0:1761315453.646301:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315453.647117:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000026030406 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307549504:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 1080us (1789us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315453.647131:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2161 00000100:00100000:3.0:1761315453.647379:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549376 00000100:00100000:3.0:1761315453.647414:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4399 00000100:00100000:3.0:1761315453.647420:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307549440:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315453.647421:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4400 00000100:00100000:1.0:1761315453.647426:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099ff5107 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307549376:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315453.647475:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000465cf088 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307549440:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 55us (1285us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315453.647482:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4399 00000100:00100000:1.0:1761315453.648472:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099ff5107 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307549376:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1042us (2169us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315453.648481:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4400 00010000:00010000:3.0:1761315453.659801:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315454.175631:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322533056:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315454.175701:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315454.176178:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533056 00000100:00100000:0.0:1761315454.176457:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2758 00000100:00100000:0.0:1761315454.176464:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322533056:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315454.176495:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315454.176981:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322533056:12345-0@lo:41:osp-pre-1-0.0 Request processed in 515us (1279us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315454.176989:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2758 00000100:00100000:2.0:1761315454.178246:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322533056:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315454.683552:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315455.708162:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315456.731912:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322533120:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315456.731933:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315456.731970:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a30fa38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322533184:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315456.732001:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315456.732010:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315456.732061:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533120 00000100:00100000:3.0:1761315456.732071:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533184 00000100:00100000:3.0:1761315456.732107:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 968 00000100:00100000:0.0:1761315456.732111:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2759 00000100:00100000:3.0:1761315456.732115:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322533184:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315456.732120:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fb5238e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322533120:12345-0@lo:41:osp-pre-0-1.0 00002000:00000024:3.0:1761315456.732146:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000004:00000024:0.0:1761315456.732150:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315456.732195:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322533184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315456.732201:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fb5238e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322533120:12345-0@lo:41:osp-pre-0-1.0 Request processed in 81us (200us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315456.732207:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 968 00000100:00100000:0.0:1761315456.732213:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2759 00000100:00100000:3.0:1761315456.732261:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322533120:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315456.732263:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a30fa38 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322533184:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315456.733405:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322533312:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315456.733420:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315456.733426:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315456.733429:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315456.733436:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3699 00000100:00080000:2.0:1761315456.733439:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315456.733444:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315456.733450:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315456.733466:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a10d07e x1846869322533504/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315456.733477:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:2.0:1761315456.733484:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315456.733485:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322533248:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315456.733490:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315456.733492:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315456.733496:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3699 00000100:00080000:2.0:1761315456.733498:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315456.733501:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315456.733514:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e376567a x1846869322533568/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315456.733521:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315456.733550:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533248 00000100:00100000:0.0:1761315456.733578:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1636 00000100:00100000:0.0:1761315456.733582:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322533248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315456.733616:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533312 00000100:00100000:0.0:1761315456.733620:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322533248:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (101us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315456.733627:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1636 00000100:00100000:1.0:1761315456.733653:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4401 00000100:00100000:1.0:1761315456.733659:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322533312:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.733673:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322533248:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315456.733715:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322533312:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315456.733723:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4401 00000100:00100000:0.0:1761315456.733740:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322533312:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315456.733757:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322533440:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.733774:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322533504:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315456.733787:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315456.733804:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315456.733811:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322533376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.733812:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315456.733842:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315456.733869:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533376 00000100:00100000:0.0:1761315456.733918:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4402 00000100:00100000:0.0:1761315456.733922:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322533376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.733953:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fafad97 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322533376:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (112us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315456.733958:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4402 00000100:00100000:0.0:1761315456.733984:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533504 00000100:00100000:0.0:1761315456.734001:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2162 00000100:00100000:0.0:1761315456.734004:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322533504:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315456.734080:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533440 00010000:02020000:0.0:1761315456.734098:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315456.734111:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2760 00000100:00100000:3.0:1761315456.734116:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322533440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.734133:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322533504:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (323us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315456.734140:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2162 00000100:00100000:3.0:1761315456.734159:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024d2fcef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322533440:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (371us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315456.734166:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2760 00000100:00080000:0.0:1761315456.734167:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315456.734172:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315456.734176:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a10d07e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322533504:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734306:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322533696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734331:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315456.734366:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315456.734371:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322533632:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734392:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315456.734416:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533696 00000100:00100000:2.0:1761315456.734436:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533632 00000100:00100000:2.0:1761315456.734451:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4403 00000100:00100000:2.0:1761315456.734455:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322533696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734487:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322533696:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (155us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315456.734493:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4403 00000100:00100000:2.0:1761315456.734504:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4404 00000100:00100000:2.0:1761315456.734507:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a10d07e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322533632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734547:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a10d07e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322533632:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (155us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315456.734556:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4404 00000100:00100000:2.0:1761315456.734582:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322533632:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734591:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322533696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734607:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322533760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734629:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315456.734654:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533760 00000100:00100000:2.0:1761315456.734675:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1458 00000100:00100000:2.0:1761315456.734678:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322533760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315456.734707:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322533760:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (78us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315456.734712:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1458 00000100:00100000:2.0:1761315456.734734:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322533760:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315456.734963:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322533376:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315456.734975:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322533440:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315456.735782:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322533568:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315456.735823:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315456.735920:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533568 00000100:00100000:1.0:1761315456.735954:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2163 00000100:00100000:1.0:1761315456.735959:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322533568:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315456.736044:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315456.739700:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322533568:12345-0@lo:8:kworker/u8:0.0 Request processed in 3731us (3868us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315456.739717:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2163 00000100:00080000:0.0:1761315456.740920:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315456.740927:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315456.740933:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322533568:0@lo:8:kworker/u8:0.0 00010000:00010000:3.0:1761315457.756275:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315458.766326:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315458.768632:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315458.768667:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315458.768703:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315458.768751:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549568 00000100:00100000:0.0:1761315458.768813:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1637 00000100:00100000:0.0:1761315458.768821:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307549568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315458.768896:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307549568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (2567us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315458.768905:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1637 00000100:00100000:1.0:1761315458.768922:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549696 00000100:00100000:1.0:1761315458.768956:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549632 00000100:00100000:0.0:1761315458.768965:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549760 00000100:00100000:3.0:1761315458.768979:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4405 00000100:00100000:3.0:1761315458.768987:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307549696:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315458.768993:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2164 00000100:00100000:0.0:1761315458.768999:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307549760:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761315458.769022:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307549696:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (397us total) trans 0 rc 0/0 00010000:02020000:0.0:1761315458.769087:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315458.769115:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307549760:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 115us (413us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315458.769122:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2164 00000100:00100000:1.0:1761315458.769316:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4406 00000100:00100000:1.0:1761315458.769323:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307549632:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315458.769375:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307549632:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 53us (709us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315458.769382:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4406 00000100:00100000:3.0:1761315458.769995:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4405 00010000:00010000:0.0:1761315458.780993:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315459.292879:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315459.293672:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322533824:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315459.293737:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315459.293896:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533824 00000100:00100000:0.0:1761315459.294923:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2761 00000100:00100000:0.0:1761315459.294932:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322533824:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315459.294969:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315459.295826:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322533824:12345-0@lo:41:osp-pre-1-0.0 Request processed in 892us (2088us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315459.295836:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2761 00000100:00100000:2.0:1761315459.295990:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322533824:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315459.803525:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315460.827593:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315461.851546:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315461.851553:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315461.851556:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315461.851563:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3704 00000100:00080000:2.0:1761315461.851567:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315461.851573:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315461.851593:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000097937f1c x1846869322534208/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315461.851606:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315461.851609:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315461.851610:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315461.851613:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3704 00000100:00080000:2.0:1761315461.851614:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315461.851617:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315461.851628:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000889dbf19 x1846869322534272/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315461.851751:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322533888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.851817:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.851840:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322534144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.851879:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.851892:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322534528:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.851914:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.851936:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315461.851941:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322533952:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.851967:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.851982:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322534336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852000:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.852032:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533888 00000100:00100000:3.0:1761315461.852071:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1761315461.852077:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322533888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852124:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322533888:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (307us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.852131:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1761315461.852162:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534528 00000100:00100000:3.0:1761315461.852185:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1459 00000100:00100000:3.0:1761315461.852188:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322534528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852217:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322534528:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (303us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.852223:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1459 00000100:00100000:3.0:1761315461.852241:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322533952 00000100:00100000:3.0:1761315461.852261:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534336 00000100:00100000:3.0:1761315461.852277:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4407 00000100:00100000:3.0:1761315461.852280:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006bf090c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322533952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852307:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006bf090c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322533952:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (340us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.852312:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4407 00000100:00100000:3.0:1761315461.852322:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4408 00000100:00100000:3.0:1761315461.852326:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba32f26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322534336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852379:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba32f26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322534336:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (379us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.852384:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4408 00000100:00100000:3.0:1761315461.852418:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb9fba07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322533952:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852429:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322534336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852436:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322533888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.852444:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322534528:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852664:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322534080:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852698:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315461.852725:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534464:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852746:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315461.852760:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315461.852765:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852785:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315461.852797:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322534400:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852814:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315461.852835:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534080 00000100:00100000:1.0:1761315461.852856:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534400 00000100:00100000:1.0:1761315461.852884:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4409 00000100:00100000:1.0:1761315461.852888:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322534080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852924:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9dc697d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322534080:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315461.852929:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4409 00000100:00100000:1.0:1761315461.852940:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4410 00000100:00100000:1.0:1761315461.852943:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322534400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.852966:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1b80cdb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322534400:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315461.852970:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4410 00000100:00100000:1.0:1761315461.853003:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322534400:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315461.853011:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322534080:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315461.853211:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322534592:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315461.853241:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.853318:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534464 00000100:00100000:3.0:1761315461.853360:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1460 00000100:00100000:3.0:1761315461.853363:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322534464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315461.853399:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322534464:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (652us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.853404:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1460 00000100:00100000:3.0:1761315461.853433:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534464:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315461.853553:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315461.853562:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322534208:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315461.853585:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315461.853598:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000889dbf19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322534272:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315461.853616:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315461.853640:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534208 00000100:00100000:0.0:1761315461.853657:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534272 00000100:00100000:0.0:1761315461.853673:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2165 00000100:00100000:0.0:1761315461.853677:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322534208:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315461.853763:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315461.853795:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322534208:12345-0@lo:8:kworker/u8:0.0 Request processed in 117us (209us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315461.853800:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2165 00000100:00100000:0.0:1761315461.853811:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2166 00000100:00100000:0.0:1761315461.853815:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322534272:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315461.853931:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534144 00000100:00100000:2.0:1761315461.853959:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534016 00000100:00100000:2.0:1761315461.853969:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534592 00000100:00100000:2.0:1761315461.853978:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2762 00000100:00100000:2.0:1761315461.853981:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054a12775 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322534144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315461.854019:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054a12775 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322534144:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (2140us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315461.854025:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2762 00000100:00100000:2.0:1761315461.854036:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2763 00000100:00100000:2.0:1761315461.854039:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322534016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315461.854069:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e88e689 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322534016:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (1284us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315461.854075:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2763 00000100:00100000:2.0:1761315461.854085:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2764 00000100:00100000:2.0:1761315461.854088:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322534592:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315461.854108:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315461.854137:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322534592:12345-0@lo:41:osp-pre-0-1.0 Request processed in 48us (896us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315461.854142:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2764 00000100:00100000:2.0:1761315461.854168:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322534144:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315461.854205:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534016:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315461.854261:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315461.854266:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315461.854269:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322534208:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315461.854417:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000030335973 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322534656:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315461.854444:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315461.854513:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322534592:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315461.854535:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534656 00000100:00100000:3.0:1761315461.854555:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 969 00000100:00100000:3.0:1761315461.854558:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322534656:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315461.854574:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315461.854731:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322534656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 170us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315461.854737:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 969 00000100:00100000:2.0:1761315461.854782:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000030335973 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322534656:0@lo:13:osp-pre-1-1.0 00010000:02020000:0.0:1761315461.854907:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315461.855084:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322534272:12345-0@lo:8:kworker/u8:0.0 Request processed in 1268us (1467us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315461.855091:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2166 00000100:00080000:2.0:1761315461.855138:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315461.855142:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315461.855145:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000889dbf19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322534272:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315462.875513:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315463.879842:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315463.879901:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315463.879944:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315463.880020:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549952 00000100:00100000:1.0:1761315463.880041:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549824 00000100:00100000:1.0:1761315463.880079:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1639 00000100:00100000:2.0:1761315463.880080:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307549888 00000100:00100000:1.0:1761315463.880088:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307549824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315463.880107:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4411 00000100:00100000:2.0:1761315463.880113:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307549952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315463.880154:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307549824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (321us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315463.880164:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307549952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (263us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315463.880172:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4411 00000100:00100000:0.0:1761315463.880191:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4412 00000100:00100000:0.0:1761315463.880198:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307549888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315463.880226:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307549888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 28us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315463.880233:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4412 00000100:00100000:1.0:1761315463.880281:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1639 00000100:00100000:1.0:1761315463.882936:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315463.884248:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550016 00000100:00100000:0.0:1761315463.885826:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2167 00000100:00100000:0.0:1761315463.885835:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ea34960 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307550016:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315463.885945:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315463.886028:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ea34960 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307550016:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 192us (3100us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315463.886041:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2167 00010000:00010000:3.0:1761315463.899860:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315464.411823:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315464.412094:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322534720:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315464.412160:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315464.412484:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534720 00000100:00100000:3.0:1761315464.412538:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2765 00000100:00100000:3.0:1761315464.412544:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322534720:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315464.412578:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315464.412633:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322534720:12345-0@lo:41:osp-pre-1-0.0 Request processed in 89us (474us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315464.412642:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2765 00000100:00100000:1.0:1761315464.412913:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322534720:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315464.923492:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315465.947520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315466.971536:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322534784:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315466.971554:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315466.971559:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315466.971563:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761315466.971568:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322534912:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315466.971569:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3709 00000100:00080000:2.0:1761315466.971572:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315466.971579:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315466.971609:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b600536 x1846869322535168/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315466.971622:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315466.971626:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315466.971628:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761315466.971632:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315466.971635:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3709 00000100:00080000:2.0:1761315466.971637:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315466.971640:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315466.971649:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d2dc936e x1846869322535232/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315466.971675:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322535296:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.971682:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315466.971698:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534976:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.971710:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315466.971713:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322535040:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.971740:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315466.971750:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315466.971761:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322535360:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.971774:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322535424:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.971775:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534912 00000100:00100000:2.0:1761315466.971787:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315466.971812:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315466.971819:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322535168:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315466.971821:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535296 00000100:00100000:0.0:1761315466.971854:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:1.0:1761315466.971861:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535040 00000100:00100000:0.0:1761315466.971864:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002bbf3014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322534848:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315466.971876:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535360 00000100:00100000:2.0:1761315466.971880:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315466.971898:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4413 00000100:00100000:2.0:1761315466.971903:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322535232:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315466.971904:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315466.971908:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aedb9ccf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322534912:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.971934:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535104:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.971937:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315466.971969:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315466.971977:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aedb9ccf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322534912:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (349us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315466.971987:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535488:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.971988:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4413 00000100:00100000:1.0:1761315466.972010:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4414 00000100:00100000:0.0:1761315466.972014:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315466.972017:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322535296:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972046:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534848 00000100:00100000:1.0:1761315466.972054:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004012701c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322535296:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315466.972061:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4414 00000100:00100000:1.0:1761315466.972078:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4415 00000100:00100000:0.0:1761315466.972084:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 970 00000100:00100000:1.0:1761315466.972085:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322535040:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972092:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322534848:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315466.972116:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315466.972120:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322535040:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (371us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315466.972127:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4415 00000100:00100000:1.0:1761315466.972141:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4416 00000100:00100000:1.0:1761315466.972144:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322535360:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.972184:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322535360:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (397us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315466.972189:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4416 00010000:00010000:3.0:1761315466.972193:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315466.972234:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322535360:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315466.972235:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534784 00000100:00100000:1.0:1761315466.972252:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315466.972262:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1640 00000100:00100000:1.0:1761315466.972264:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322535552:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315466.972270:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322534784:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.972293:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315466.972307:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322534784:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (632us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315466.972314:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1640 00000100:00100000:0.0:1761315466.972375:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322534848:12345-0@lo:13:osp-pre-1-1.0 Request processed in 284us (473us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315466.972383:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 970 00000100:00100000:0.0:1761315466.972419:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002bbf3014 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322534848:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315466.972427:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322535040:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972434:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000abeba833 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322534784:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972477:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322535296:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972486:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322534912:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972520:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535424 00000100:00100000:0.0:1761315466.972540:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535168 00000100:00100000:0.0:1761315466.972551:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535232 00000100:00100000:0.0:1761315466.972558:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535488 00000100:00100000:0.0:1761315466.972568:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1461 00000100:00100000:0.0:1761315466.972571:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322535424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972603:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322535424:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (794us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315466.972609:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1461 00000100:00100000:0.0:1761315466.972620:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1462 00000100:00100000:0.0:1761315466.972623:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315466.972646:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535488:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (634us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315466.972651:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1462 00000100:00100000:0.0:1761315466.972662:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2168 00000100:00100000:0.0:1761315466.972665:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322535168:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315466.972749:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315466.972821:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322535168:12345-0@lo:8:kworker/u8:2.0 Request processed in 155us (944us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315466.972841:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2168 00000100:00100000:0.0:1761315466.972853:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2169 00000100:00100000:0.0:1761315466.972855:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322535232:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315466.972929:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315466.972937:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322534976 00000100:00100000:0.0:1761315466.972967:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322535232:12345-0@lo:8:kworker/u8:2.0 Request processed in 108us (1033us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315466.972971:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535104 00000100:00100000:0.0:1761315466.972975:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2169 00000100:00100000:2.0:1761315466.972986:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535552 00000100:00100000:2.0:1761315466.973002:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2766 00000100:00100000:1.0:1761315466.973008:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535488:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.973008:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322534976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315466.973029:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322535424:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.973055:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322534976:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (1317us total) trans 0 rc 0/0 00000100:00080000:0.0:1761315466.973062:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315466.973063:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2766 00000100:00080000:0.0:1761315466.973068:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315466.973074:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322535168:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315466.973085:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2767 00000100:00080000:0.0:1761315466.973087:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315466.973091:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315466.973094:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535104:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.973095:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2768 00000100:00100000:0.0:1761315466.973098:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322535232:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315466.973105:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322535552:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315466.973131:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315466.973142:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535104:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (1174us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315466.973150:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2767 00000100:00100000:0.0:1761315466.973159:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535104:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315466.973168:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000480f4338 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322535552:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (874us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315466.973175:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2768 00000100:00100000:1.0:1761315466.973203:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322535552:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315466.973212:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322534976:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315467.997061:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315468.999806:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315468.999875:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315468.999891:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550144 00000100:00100000:2.0:1761315468.999919:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315468.999935:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550080 00000100:00100000:1.0:1761315468.999953:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550208 00000100:00100000:1.0:1761315468.999980:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4417 00000100:00100000:1.0:1761315468.999987:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307550144:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315468.999990:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1641 00000100:00100000:3.0:1761315468.999999:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307550080:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315469.000057:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307550144:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 70us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315469.000064:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4417 00000100:00100000:1.0:1761315469.000077:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4418 00000100:00100000:1.0:1761315469.000081:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307550208:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315469.000149:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307550080:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 148us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315469.000162:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1641 00000100:00100000:3.0:1761315469.001063:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315469.001289:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307550208:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1204us (1370us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315469.001299:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4418 00000100:00100000:0.0:1761315469.002000:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550272 00000100:00100000:0.0:1761315469.002050:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2170 00000100:00100000:0.0:1761315469.002054:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307550272:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315469.002143:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315469.003500:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3f71b47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307550272:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 1439us (2434us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315469.003517:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2170 00010000:00010000:0.0:1761315469.019520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315469.532072:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315469.534445:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008135f598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322535616:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315469.534517:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315469.534650:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535616 00000100:00100000:3.0:1761315469.534716:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2769 00000100:00100000:3.0:1761315469.534722:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322535616:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315469.534755:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315469.534832:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322535616:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (291us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315469.534845:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2769 00000100:00100000:1.0:1761315469.534967:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008135f598 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322535616:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315470.044726:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315471.067581:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315472.091652:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315472.091677:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322535680:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315472.091749:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315472.092958:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535680 00000100:00080000:3.0:1761315472.092995:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315472.093001:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315472.093004:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761315472.093009:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2770 00000100:00080000:3.0:1761315472.093011:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3714 00000100:00080000:3.0:1761315472.093014:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315472.093015:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322535680:12345-0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315472.093019:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315472.093038:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000026030406 x1846869322536000/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000004:00000024:2.0:1761315472.093047:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:3.0:1761315472.093050:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315472.093053:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315472.093054:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315472.093057:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3714 00000100:00080000:3.0:1761315472.093058:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315472.093061:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315472.093069:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bfaae214 x1846869322536064/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315472.093098:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000696bc59 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322535680:12345-0@lo:41:osp-pre-0-1.0 Request processed in 82us (1350us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315472.093104:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2770 00000100:00100000:1.0:1761315472.093184:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322535872:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093221:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093237:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322536256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093262:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093282:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315472.093286:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc7bc95e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535936:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093286:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322535808:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093310:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315472.093312:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315472.093325:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322536320:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093325:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322536192:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315472.093359:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093359:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315472.093374:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315472.093379:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322535744:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093384:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536256 00000100:00100000:0.0:1761315472.093405:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093410:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536320 00000100:00100000:0.0:1761315472.093417:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0c7468 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322536128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093430:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1463 00000100:00100000:1.0:1761315472.093434:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002bbf3014 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322536256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315472.093436:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315472.093457:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322536000:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093465:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002bbf3014 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322536256:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (204us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315472.093471:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1463 00000100:00100000:0.0:1761315472.093476:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093484:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536000 00000100:00100000:0.0:1761315472.093488:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322536064:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093499:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1464 00000100:00100000:1.0:1761315472.093502:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322536320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315472.093506:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315472.093528:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322536320:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315472.093533:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1464 00000100:00100000:1.0:1761315472.093542:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536064 00000100:00100000:1.0:1761315472.093551:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2171 00000100:00100000:0.0:1761315472.093554:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2172 00000100:00100000:1.0:1761315472.093555:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322536000:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315472.093558:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322536064:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315472.093712:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315472.093716:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315472.093742:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322536000:12345-0@lo:8:kworker/u8:0.0 Request processed in 186us (265us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315472.093748:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2171 00000100:00100000:1.0:1761315472.093766:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535872 00000100:00100000:2.0:1761315472.093776:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535808 00000100:00100000:1.0:1761315472.093786:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536192 00000100:00100000:2.0:1761315472.093811:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536128 00000100:00100000:2.0:1761315472.093822:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4419 00000100:00100000:2.0:1761315472.093825:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030335973 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322535872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315472.093853:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030335973 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322535872:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (632us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315472.093858:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4419 00000100:00100000:2.0:1761315472.093866:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4420 00000100:00100000:2.0:1761315472.093869:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322535808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315472.093900:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322535808:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (590us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315472.093905:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4420 00000100:00100000:2.0:1761315472.093912:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4421 00000100:00100000:2.0:1761315472.093915:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322536128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315472.093931:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322536128:12345-0@lo:400:kworker/u8:0.0 Request processed in 16us (496us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315472.093935:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4421 00000100:00100000:2.0:1761315472.093958:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535744 00000100:00100000:2.0:1761315472.093979:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1642 00000100:00100000:1.0:1761315472.093981:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4422 00000100:00100000:2.0:1761315472.093982:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322535744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.093984:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322536192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094008:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322536192:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (665us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315472.094008:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322535744:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (602us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315472.094014:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4422 00000100:00100000:2.0:1761315472.094014:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1642 00000100:00100000:1.0:1761315472.094066:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322535936 00000100:00100000:1.0:1761315472.094087:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2771 00000100:00100000:1.0:1761315472.094091:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094124:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322535936:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (811us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315472.094130:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2771 00010000:00010000:1.0:1761315472.094160:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315472.094175:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322536384:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315472.094184:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322535744:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315472.094195:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0c7468 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322536128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094200:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315472.094203:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc7bc95e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322535936:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315472.094210:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a254dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322536320:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094223:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322535680:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315472.094244:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322535808:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094252:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536384 00000100:00100000:3.0:1761315472.094253:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322536192:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315472.094262:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322535872:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315472.094269:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084ec238b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322536256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315472.094272:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 971 00000100:00100000:1.0:1761315472.094275:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322536384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315472.094292:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315472.094322:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322536384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (122us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315472.094327:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 971 00000100:00100000:3.0:1761315472.094337:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322536384:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315472.094632:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322536064:12345-0@lo:8:kworker/u8:0.0 Request processed in 1073us (1125us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315472.094640:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2172 00000100:00080000:2.0:1761315472.094779:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315472.094783:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315472.094787:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322536000:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315472.094807:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315472.094810:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315472.094812:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bfaae214 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322536064:0@lo:8:kworker/u8:0.0 00010000:00010000:3.0:1761315473.116317:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315474.121434:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315474.121481:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315474.121584:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550400 00000100:00100000:3.0:1761315474.121654:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4423 00000100:00100000:3.0:1761315474.121673:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307550400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315474.121772:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307550400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 109us (347us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315474.121789:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4423 00000100:00100000:1.0:1761315474.123290:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550528 00000100:00100000:1.0:1761315474.123332:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2173 00000100:00100000:1.0:1761315474.123338:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307550528:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315474.123461:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315474.123520:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307550528:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 181us (2039us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315474.123530:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2173 00000100:00100000:2.0:1761315474.123875:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315474.125566:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315474.126029:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550464 00000100:00100000:3.0:1761315474.126528:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550336 00000100:00100000:3.0:1761315474.126568:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1643 00000100:00100000:3.0:1761315474.126576:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307550336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315474.126654:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307550336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 78us (2780us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315474.126671:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1643 00000100:00100000:0.0:1761315474.126948:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4424 00000100:00100000:0.0:1761315474.126954:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307550464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315474.127011:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307550464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 56us (1451us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315474.127018:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4424 00010000:00010000:0.0:1761315474.139492:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315474.652441:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315474.652885:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322536448:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315474.652951:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315474.653078:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536448 00000100:00100000:1.0:1761315474.653131:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2772 00000100:00100000:1.0:1761315474.653138:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322536448:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315474.653182:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315474.653241:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322536448:12345-0@lo:41:osp-pre-1-0.0 Request processed in 102us (290us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315474.653248:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2772 00000100:00100000:0.0:1761315474.654236:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322536448:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315475.163568:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315476.188940:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315477.211736:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315477.211791:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322536512:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315477.211864:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1761315477.212422:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315477.212907:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536512 00000100:00100000:3.0:1761315477.212955:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2773 00000100:00100000:3.0:1761315477.212962:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322536512:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315477.212994:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315477.213048:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322536512:12345-0@lo:41:osp-pre-0-1.0 Request processed in 85us (1185us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315477.213055:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2773 00000100:00080000:2.0:1761315477.213220:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315477.213224:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315477.213227:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315477.213232:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3719 00000100:00080000:2.0:1761315477.213234:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315477.213238:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315477.213254:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d0629bf1 x1846869322536832/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315477.213269:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315477.213272:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315477.213273:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315477.213276:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3719 00000100:00080000:2.0:1761315477.213277:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315477.213280:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315477.213288:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e4a4d687 x1846869322536896/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315477.213329:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322536832:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315477.213386:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315477.213407:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322536896:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315477.213429:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315477.213457:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536832 00000100:00100000:0.0:1761315477.213475:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322536768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.213476:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536896 00000100:00100000:2.0:1761315477.213488:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2174 00000100:00100000:2.0:1761315477.213492:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322536832:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315477.213512:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315477.213524:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322537152:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.213549:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315477.213573:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315477.213577:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322536704:0@lo:400:kworker/u8:2.0 00010000:02020000:2.0:1761315477.213580:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315477.213612:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322536832:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (226us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315477.213614:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315477.213618:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2174 00000100:00100000:2.0:1761315477.213629:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537152 00000100:00100000:0.0:1761315477.213633:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b6ed247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322537088:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.213654:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315477.213685:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537088 00000100:00100000:0.0:1761315477.213715:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1465 00000100:00100000:0.0:1761315477.213719:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322537088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.213778:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322537088:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (123us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315477.213784:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1465 00000100:00100000:0.0:1761315477.213798:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2175 00000100:00100000:0.0:1761315477.213801:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322536896:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315477.213895:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315477.213929:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322536896:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (499us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315477.213934:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2175 00000100:00100000:0.0:1761315477.213959:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536704 00000100:00100000:0.0:1761315477.213982:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4425 00000100:00100000:0.0:1761315477.213993:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322536704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.214030:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322536704:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315477.214036:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4425 00000100:00100000:0.0:1761315477.214079:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322536704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.214090:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b6ed247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322537088:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315477.214133:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315477.214137:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315477.214139:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322536832:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315477.214147:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315477.214149:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315477.214151:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322536896:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315477.214881:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536768 00000100:00100000:1.0:1761315477.214922:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2774 00000100:00100000:1.0:1761315477.214928:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322536768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315477.214977:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322536768:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (1465us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315477.214983:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2774 00000100:00100000:1.0:1761315477.215034:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322536512:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315477.215059:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f576ac9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322536768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.215158:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1466 00000100:00100000:2.0:1761315477.215162:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322537152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.215198:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322537152:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (1648us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315477.215205:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1466 00000100:00100000:3.0:1761315477.215284:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322536576:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315477.215400:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315477.215416:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322536960:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315477.215441:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315477.215455:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322537216:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315477.215478:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315477.215495:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322536640:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315477.215507:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537216 00000100:00100000:0.0:1761315477.215530:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315477.215531:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 972 00000100:00100000:3.0:1761315477.215535:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322537216:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315477.215553:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322537024:0@lo:400:kworker/u8:2.0 00002000:00000024:3.0:1761315477.215556:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315477.215627:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322537152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315477.215656:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536960 00000100:00100000:1.0:1761315477.215678:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536640 00000100:00100000:1.0:1761315477.215696:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4426 00000100:00100000:1.0:1761315477.215700:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322536960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315477.215969:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322537216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 433us (490us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315477.215976:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 972 00000100:00100000:3.0:1761315477.216011:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322537216:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315477.216048:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4427 00000100:00100000:3.0:1761315477.216052:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322536640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315477.216080:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322536640:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (551us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315477.216086:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4427 00000100:00100000:0.0:1761315477.216142:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315477.216167:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322536576 00000100:00100000:2.0:1761315477.216190:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1644 00000100:00100000:0.0:1761315477.216191:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322536640:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.216194:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322536576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.216232:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322536576:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (832us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315477.216238:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1644 00000100:00100000:0.0:1761315477.216257:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005abee3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322536576:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.216266:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537024 00000100:00100000:2.0:1761315477.216283:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4428 00000100:00100000:2.0:1761315477.216287:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322537024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315477.216328:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322537024:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315477.216333:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4428 00000100:00100000:1.0:1761315477.216449:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322536960:12345-0@lo:400:kworker/u8:2.0 Request processed in 748us (1007us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315477.216456:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4426 00000100:00100000:3.0:1761315477.216523:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322536960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315477.216618:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322537024:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315478.236862:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315479.240920:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315479.242177:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550656 00000100:00100000:1.0:1761315479.242243:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4429 00000100:00100000:1.0:1761315479.242249:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307550656:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315479.242335:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307550656:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 85us (1422us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315479.242356:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4429 00000100:00100000:2.0:1761315479.242642:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315479.243934:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315479.244718:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550592 00000100:00100000:0.0:1761315479.245945:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550720 00000100:00100000:0.0:1761315479.245998:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4430 00000100:00100000:0.0:1761315479.246005:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307550720:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315479.246079:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307550720:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 74us (2147us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315479.246087:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4430 00000100:00100000:2.0:1761315479.246455:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315479.247058:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1645 00000100:00100000:1.0:1761315479.247065:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307550592:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315479.247108:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307550592:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 43us (4468us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315479.247114:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1645 00000100:00100000:0.0:1761315479.248522:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550784 00000100:00100000:0.0:1761315479.248772:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2176 00000100:00100000:0.0:1761315479.248778:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307550784:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315479.248875:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315479.253171:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307550784:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 4385us (6711us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315479.253186:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2176 00010000:00010000:3.0:1761315479.262258:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315479.771573:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322537280:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315479.771639:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315479.771752:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537280 00000100:00100000:0.0:1761315479.771808:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2775 00000100:00100000:0.0:1761315479.771814:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322537280:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315479.771846:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315479.772813:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322537280:12345-0@lo:41:osp-pre-1-0.0 Request processed in 996us (1173us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315479.772823:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2775 00000100:00100000:3.0:1761315479.772916:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322537280:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315480.284026:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315481.307556:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315482.331506:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315482.331528:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322537344:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315482.331599:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315482.331667:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537344 00000100:00100000:0.0:1761315482.331775:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2776 00000100:00100000:0.0:1761315482.331782:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322537344:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315482.331812:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:2.0:1761315482.331928:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315482.331933:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315482.331936:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315482.331941:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3724 00000100:00080000:2.0:1761315482.331943:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315482.331947:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315482.331967:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000009cdcdea x1846869322537664/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315482.331981:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315482.331983:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315482.331984:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315482.332056:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3724 00000100:00080000:2.0:1761315482.332059:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315482.332071:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315482.332083:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000072fdd2e7 x1846869322537728/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315482.332119:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322537600:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332145:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332159:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322537984:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332178:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332192:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315482.332196:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322537408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332216:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332226:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c9b8c817 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322537792:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332244:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332265:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537600 00000100:00100000:2.0:1761315482.332286:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2777 00000100:00100000:2.0:1761315482.332290:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322537600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332320:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc01d0b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322537600:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (176us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315482.332327:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2777 00000100:00100000:2.0:1761315482.332380:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322537600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.332392:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322537472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332395:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315482.332404:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315482.332429:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332434:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315482.332455:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e57767e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322537856:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332474:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322537536:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.332484:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315482.332505:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315482.332524:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537792 00000100:00100000:2.0:1761315482.332525:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df11d5eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322537920:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315482.332547:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315482.332558:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537472 00000100:00100000:1.0:1761315482.332597:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537856 00000100:00100000:0.0:1761315482.332615:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322537344:12345-0@lo:41:osp-pre-0-1.0 Request processed in 833us (1019us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315482.332627:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2776 00000100:00100000:0.0:1761315482.332645:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322537664:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315482.332667:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315482.332678:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322537728:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315482.332693:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315482.332729:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537984 00000100:00100000:0.0:1761315482.332751:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537920 00000100:00100000:0.0:1761315482.332764:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537664 00000100:00100000:0.0:1761315482.332771:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537728 00000100:00100000:0.0:1761315482.332779:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1467 00000100:00100000:0.0:1761315482.332782:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322537984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315482.332812:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f31517f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322537984:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (634us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315482.332817:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1467 00000100:00100000:0.0:1761315482.332826:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1468 00000100:00100000:0.0:1761315482.332830:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322537920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315482.332852:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322537920:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315482.332856:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1468 00000100:00100000:0.0:1761315482.332865:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2177 00000100:00100000:0.0:1761315482.332880:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322537664:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315482.332965:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315482.332992:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322537664:12345-0@lo:8:kworker/u8:2.0 Request processed in 111us (325us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315482.332997:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2177 00000100:00100000:0.0:1761315482.333007:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2178 00000100:00100000:0.0:1761315482.333010:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc64cfd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322537728:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315482.333236:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538048 00000100:00100000:3.0:1761315482.333271:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 973 00000100:00100000:3.0:1761315482.333277:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322538048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315482.333300:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315482.333339:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322538048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (906us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315482.333396:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 973 00000100:00100000:3.0:1761315482.333427:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537408 00000100:00100000:3.0:1761315482.333454:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1646 00000100:00100000:3.0:1761315482.333459:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322537408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315482.333495:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322537408:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (1278us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315482.333502:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1646 00010000:00010000:3.0:1761315482.333542:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:02020000:0.0:1761315482.333576:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315482.333601:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc64cfd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322537728:12345-0@lo:8:kworker/u8:2.0 Request processed in 590us (907us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315482.333607:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2178 00000100:00100000:1.0:1761315482.333760:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322537536 00000100:00100000:1.0:1761315482.333778:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4431 00000100:00100000:1.0:1761315482.333782:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322537792:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.333812:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322537792:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (1567us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315482.333817:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4431 00000100:00100000:1.0:1761315482.333827:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4432 00000100:00100000:1.0:1761315482.333830:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322537472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.333855:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322537472:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (1428us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315482.333860:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4432 00000100:00100000:1.0:1761315482.333879:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4433 00000100:00100000:1.0:1761315482.333883:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322537856:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.333904:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322537856:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (1423us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315482.333910:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4433 00000100:00100000:1.0:1761315482.333918:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4434 00000100:00100000:1.0:1761315482.333920:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322537536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.333938:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322537536:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (1436us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315482.333943:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4434 00000100:00100000:1.0:1761315482.333987:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322537472:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.333996:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e57767e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322537856:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.334049:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322537344:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315482.334090:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322537536:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.334099:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df11d5eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322537920:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315482.334126:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315482.334129:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315482.334132:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000009cdcdea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322537664:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315482.334139:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315482.334141:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315482.334143:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322537728:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315482.334175:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315482.334183:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322537408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.334191:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c9b8c817 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322537792:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315482.334200:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000738cd15f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322537984:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761315483.355595:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315484.364388:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315484.364441:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315484.364483:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315484.364533:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315484.364579:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551040 00000100:00100000:2.0:1761315484.364628:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550848 00000100:00100000:0.0:1761315484.364632:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2179 00000100:00100000:0.0:1761315484.364640:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307551040:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761315484.364664:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550976 00000100:00100000:2.0:1761315484.364665:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1647 00000100:00100000:2.0:1761315484.364670:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307550848:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315484.364691:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307550912 00000100:00100000:3.0:1761315484.364720:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4435 00000100:00100000:3.0:1761315484.364724:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307550976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315484.364749:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307550848:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 78us (308us total) trans 0 rc 0/0 00010000:02020000:0.0:1761315484.364752:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315484.364758:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307550976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (275us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315484.364760:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1647 00000100:00100000:3.0:1761315484.364764:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4435 00000100:00100000:3.0:1761315484.364777:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4436 00000100:00100000:3.0:1761315484.364782:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307550912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315484.364790:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307551040:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 149us (409us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315484.364798:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2179 00000100:00100000:3.0:1761315484.364809:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307550912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (275us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315484.364817:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4436 00010000:00010000:3.0:1761315484.383987:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315484.892594:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315484.892616:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538112:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315484.892677:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315484.892767:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538112 00000100:00100000:0.0:1761315484.892813:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2778 00000100:00100000:0.0:1761315484.892819:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538112:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315484.892848:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315484.892898:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538112:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (221us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315484.892905:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2778 00000100:00100000:3.0:1761315484.892951:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fba44d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538112:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315485.403543:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315486.434724:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315487.451548:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315487.451555:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315487.451558:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315487.451565:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3729 00000100:00080000:2.0:1761315487.451569:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315487.451574:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315487.451591:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000671ff9a0 x1846869322538496/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315487.451604:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315487.451607:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315487.451608:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315487.451611:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3729 00000100:00080000:2.0:1761315487.451612:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315487.451614:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315487.451622:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000007c7b302 x1846869322538560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315487.451758:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322538240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.451823:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315487.451860:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322538624:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.451879:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315487.451896:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315487.451900:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.451920:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315487.451934:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322538688:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.451950:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315487.451982:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538240 00000100:00100000:0.0:1761315487.452017:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538624 00000100:00100000:0.0:1761315487.452031:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538688 00000100:00100000:0.0:1761315487.452046:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4437 00000100:00100000:0.0:1761315487.452050:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005abee3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322538240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452112:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005abee3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322538240:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315487.452118:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4437 00000100:00100000:0.0:1761315487.452134:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4438 00000100:00100000:0.0:1761315487.452137:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322538624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452160:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322538624:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (281us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315487.452165:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4438 00000100:00100000:0.0:1761315487.452175:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4439 00000100:00100000:0.0:1761315487.452178:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322538688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452199:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322538688:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315487.452207:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4439 00000100:00100000:0.0:1761315487.452226:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538304 00000100:00100000:0.0:1761315487.452249:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2779 00000100:00100000:0.0:1761315487.452252:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322538304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452292:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322538304:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (371us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315487.452297:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2779 00000100:00100000:0.0:1761315487.452383:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452393:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036336669 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322538688:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452401:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322538240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.452409:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322538624:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315487.452900:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315487.452913:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322538176:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.452950:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315487.452975:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538432:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315487.452993:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538176 00000100:00100000:3.0:1761315487.452997:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315487.453022:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1648 00000100:00100000:3.0:1761315487.453022:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538816:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315487.453026:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322538176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453046:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315487.453058:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322538176:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (108us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315487.453063:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1648 00000100:00100000:3.0:1761315487.453069:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000aa10021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322538176:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453077:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315487.453082:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322538368:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453103:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315487.453127:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322538752:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315487.453143:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538368 00000100:00100000:3.0:1761315487.453150:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315487.453166:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4440 00000100:00100000:2.0:1761315487.453170:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322538368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453176:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538816 00000100:00100000:3.0:1761315487.453196:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538752 00000100:00100000:2.0:1761315487.453204:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322538368:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (101us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315487.453209:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1469 00000100:00100000:2.0:1761315487.453210:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4440 00000100:00100000:3.0:1761315487.453212:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315487.453233:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322538368:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453239:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538816:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (193us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315487.453242:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538816:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453245:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1469 00000100:00100000:3.0:1761315487.453255:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1470 00000100:00100000:3.0:1761315487.453258:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322538752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315487.453283:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfaae214 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322538752:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (133us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315487.453288:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1470 00000100:00100000:2.0:1761315487.453296:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322538752:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315487.453703:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538432 00000100:00100000:0.0:1761315487.453731:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2780 00000100:00100000:0.0:1761315487.453735:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538432:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315487.453750:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322538496:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315487.453769:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322538432:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (771us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315487.453774:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2780 00000100:00100000:1.0:1761315487.453777:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315487.453793:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322538560:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315487.453803:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538496 00000100:00100000:1.0:1761315487.453814:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315487.453819:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538560 00000100:00100000:0.0:1761315487.453827:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2180 00000100:00100000:0.0:1761315487.453830:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322538496:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315487.453836:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2181 00000100:00100000:1.0:1761315487.453839:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322538560:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315487.454005:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315487.454005:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315487.454032:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322538560:12345-0@lo:8:kworker/u8:0.0 Request processed in 191us (218us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315487.454038:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2181 00000100:00100000:2.0:1761315487.454071:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322538432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315487.454082:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315487.454089:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322538880:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315487.454114:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315487.454192:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315487.454199:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538944:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315487.454209:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538880 00000100:00100000:2.0:1761315487.454226:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315487.454229:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2781 00000100:00100000:3.0:1761315487.454232:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322538880:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315487.454253:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315487.454263:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322538944 00000100:00100000:2.0:1761315487.454285:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 974 00000100:00100000:3.0:1761315487.454285:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000738cd15f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322538880:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315487.454289:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322538944:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315487.454291:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2781 00002000:00000024:2.0:1761315487.454306:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315487.454319:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322538880:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315487.454336:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322538944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (110us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315487.454360:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 974 00000100:00100000:3.0:1761315487.454368:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322538944:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315487.455281:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322538496:12345-0@lo:8:kworker/u8:0.0 Request processed in 1447us (1499us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315487.455291:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2180 00000100:00080000:1.0:1761315487.457109:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315487.457120:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315487.457127:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322538496:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315487.457140:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315487.457142:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315487.457144:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322538560:0@lo:8:kworker/u8:0.0 00010000:00010000:2.0:1761315488.476647:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315489.481054:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315489.482842:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315489.482869:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315489.482936:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551104 00000100:00100000:0.0:1761315489.482995:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1649 00000100:00100000:0.0:1761315489.483002:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307551104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315489.483067:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307551104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 65us (229us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315489.483074:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1649 00000100:00100000:2.0:1761315489.483304:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551168 00000100:00100000:2.0:1761315489.483384:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4441 00000100:00100000:2.0:1761315489.483389:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c9b8c817 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307551168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315489.483427:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c9b8c817 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307551168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 37us (2383us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315489.483437:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4441 00000100:00100000:1.0:1761315489.483983:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315489.484003:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551296 00000100:00100000:1.0:1761315489.484020:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2182 00000100:00100000:1.0:1761315489.484025:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e57767e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307551296:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315489.484112:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315489.484158:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e57767e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307551296:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 132us (176us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315489.484168:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2182 00000100:00100000:2.0:1761315489.484230:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551232 00000100:00100000:2.0:1761315489.484263:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4442 00000100:00100000:2.0:1761315489.484268:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307551232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315489.484296:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df11d5eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307551232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 28us (1427us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315489.484303:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4442 00010000:00010000:2.0:1761315489.499999:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315490.018692:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d14b1c65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322539008:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315490.018761:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315490.018861:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539008 00000100:00100000:1.0:1761315490.018915:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2782 00000100:00100000:1.0:1761315490.018921:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322539008:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315490.018954:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315490.019007:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322539008:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315490.019014:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2782 00000100:00100000:0.0:1761315490.019097:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d14b1c65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322539008:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315490.523584:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315491.549780:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315492.571531:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315492.571717:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315492.571723:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315492.571726:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315492.571734:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3735 00000100:00080000:3.0:1761315492.571738:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315492.571744:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315492.571766:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000011d94811 x1846869322539456/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315492.571770:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322539136:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315492.571788:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315492.571792:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315492.571793:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315492.571797:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3735 00000100:00080000:3.0:1761315492.571798:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315492.571801:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315492.571814:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000034d7daff x1846869322539520/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315492.571837:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315492.571865:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539392:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315492.571894:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.571948:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539136 00000100:00100000:3.0:1761315492.571982:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322539264:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.571995:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4443 00000100:00100000:0.0:1761315492.572002:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322539136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.572013:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.572058:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322539136:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.572065:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4443 00000100:00100000:1.0:1761315492.572493:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322539776:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572521:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315492.572552:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084e45549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322539648:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572556:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f482099 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322539136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572578:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315492.572583:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315492.572586:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539200:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572609:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315492.572624:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003d52d58b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322539584:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315492.572657:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539776 00000100:00100000:1.0:1761315492.572667:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539392 00000100:00100000:2.0:1761315492.572693:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1471 00000100:00100000:2.0:1761315492.572699:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322539776:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572702:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539200 00000100:00100000:1.0:1761315492.572724:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2783 00000100:00100000:1.0:1761315492.572730:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322539392:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315492.572743:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322539776:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315492.572750:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1471 00000004:00000024:1.0:1761315492.572756:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315492.572775:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2784 00000100:00100000:2.0:1761315492.572780:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322539200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315492.572795:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322539392:12345-0@lo:41:osp-pre-0-1.0 Request processed in 65us (900us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315492.572802:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2783 00000100:00100000:2.0:1761315492.572817:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322539200:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (207us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315492.572825:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2784 00000100:00100000:0.0:1761315492.573138:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539264 00000100:00100000:0.0:1761315492.573167:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539648 00000100:00100000:3.0:1761315492.573171:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:3.0:1761315492.573180:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322539072:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573200:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4444 00000100:00100000:0.0:1761315492.573205:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322539264:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573210:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315492.573229:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc7bc95e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322539328:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573243:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322539264:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1230us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.573250:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4444 00000100:00100000:3.0:1761315492.573258:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.573266:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4445 00000100:00100000:0.0:1761315492.573271:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322539648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573277:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322539712:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573319:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322539648:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (738us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315492.573327:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.573329:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4445 00000100:00100000:0.0:1761315492.573370:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539328 00000100:00100000:3.0:1761315492.573375:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fb56f29f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322539264:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573391:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084e45549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322539648:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573397:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2785 00000100:00100000:0.0:1761315492.573402:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322539328:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573421:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322539456:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573442:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322539328:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.573449:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2785 00000100:00100000:3.0:1761315492.573454:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315492.573473:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322539520:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573490:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc7bc95e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322539328:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573505:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315492.573540:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539072 00000100:00100000:3.0:1761315492.573569:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1650 00000100:00100000:3.0:1761315492.573574:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322539072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315492.573611:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322539072:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315492.573617:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1650 00000100:00100000:0.0:1761315492.573641:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539712 00000100:00100000:0.0:1761315492.573827:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539456 00000100:00100000:0.0:1761315492.573840:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539520 00000100:00100000:0.0:1761315492.573853:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1472 00000100:00100000:0.0:1761315492.573856:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322539712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.573886:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322539712:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (562us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.573892:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1472 00000100:00100000:0.0:1761315492.573905:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2183 00000100:00100000:0.0:1761315492.573907:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322539456:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315492.573990:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315492.574020:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084e45549 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322539456:12345-0@lo:8:kworker/u8:0.0 Request processed in 111us (569us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315492.574025:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2183 00000100:00100000:0.0:1761315492.574035:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2184 00000100:00100000:0.0:1761315492.574038:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322539520:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315492.574112:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315492.574135:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322539520:12345-0@lo:8:kworker/u8:0.0 Request processed in 95us (631us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315492.574141:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2184 00000100:00100000:0.0:1761315492.574169:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.574210:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322539776:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.574223:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539392:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315492.574235:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315492.574243:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315492.574266:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315492.574287:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cea502e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539200:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.574306:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539584 00000100:00100000:0.0:1761315492.574324:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4446 00000100:00100000:0.0:1761315492.574328:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a4fe2a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322539584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315492.574384:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a4fe2a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322539584:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (213us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.574389:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4446 00000100:00100000:0.0:1761315492.574409:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539840 00000100:00100000:0.0:1761315492.574430:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 975 00000100:00100000:0.0:1761315492.574433:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fba44d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322539840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315492.574451:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315492.574478:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fba44d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322539840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (212us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315492.574486:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 975 00000100:00100000:0.0:1761315492.574525:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315492.574536:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003d52d58b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322539584:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315492.575446:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315492.575453:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315492.575454:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000026030406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322539072:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315492.575461:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011d94811 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322539456:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315492.575474:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b2151c9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322539712:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315492.575476:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315492.575480:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315492.575485:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322539520:0@lo:8:kworker/u8:0.0 00010000:00010000:1.0:1761315493.595604:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315494.601120:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315494.602555:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551360 00000100:00100000:0.0:1761315494.603030:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1651 00000100:00100000:0.0:1761315494.603038:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307551360:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315494.603122:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307551360:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 84us (2010us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315494.603129:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1651 00000100:00100000:2.0:1761315494.604854:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315494.604918:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315494.605123:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315494.605172:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551488 00000100:00100000:1.0:1761315494.605218:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551424 00000100:00100000:1.0:1761315494.605235:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4447 00000100:00100000:1.0:1761315494.605241:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000daaf5aa6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307551488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315494.605316:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000daaf5aa6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307551488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (467us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315494.605323:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4447 00000100:00100000:1.0:1761315494.607367:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4448 00000100:00100000:1.0:1761315494.607379:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307551424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315494.607436:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001339376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307551424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 59us (2517us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315494.607442:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4448 00000100:00100000:0.0:1761315494.607669:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551552 00000100:00100000:0.0:1761315494.607710:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2185 00000100:00100000:0.0:1761315494.607715:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307551552:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315494.607805:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315494.607838:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307551552:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 121us (2716us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315494.607843:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2185 00010000:00010000:2.0:1761315494.621927:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315495.131580:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315495.131836:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322539904:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315495.131903:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315495.132029:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539904 00000100:00100000:0.0:1761315495.132083:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2786 00000100:00100000:0.0:1761315495.132088:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322539904:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315495.132120:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315495.132169:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322539904:12345-0@lo:41:osp-pre-1-0.0 Request processed in 80us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315495.132176:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2786 00000100:00100000:3.0:1761315495.132255:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322539904:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315495.643728:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315496.667744:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315497.691555:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315497.691579:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322540096:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315497.691658:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315497.691706:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539968:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315497.691743:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315497.691776:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322540160:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315497.691807:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1761315497.691845:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315497.691864:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315497.691873:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322539968 00000100:00080000:3.0:1761315497.691879:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315497.691882:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315497.691888:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3740 00000100:00080000:3.0:1761315497.691890:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315497.691896:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315497.691915:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 976 00000100:00100000:3.0:1761315497.691916:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e9965e9b x1846869322540352/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315497.691924:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322539968:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315497.691937:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315497.691943:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315497.691944:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315497.691948:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3740 00000100:00080000:3.0:1761315497.691950:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315497.691953:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00002000:00000024:2.0:1761315497.691956:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315497.691966:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003226e4c2 x1846869322540416/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315497.691996:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322539968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (253us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315497.692004:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 976 00000100:00100000:3.0:1761315497.692014:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322540224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315497.692027:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322540352:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315497.692047:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315497.692061:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315497.692067:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322540608:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315497.692083:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322540416:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315497.692092:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315497.692113:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315497.692115:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315497.692135:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322540288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315497.692146:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540352 00000100:00100000:3.0:1761315497.692163:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315497.692169:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540608 00000100:00100000:1.0:1761315497.692196:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0629bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322539968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315497.692199:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540416 00000100:00100000:3.0:1761315497.692205:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540160 00000100:00100000:1.0:1761315497.692206:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322540032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315497.692219:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1473 00000100:00100000:2.0:1761315497.692225:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322540608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.692241:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540224 00000100:00100000:1.0:1761315497.692245:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315497.692271:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4449 00000100:00100000:2.0:1761315497.692276:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322540608:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.692278:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049074e99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322540160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315497.692282:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322540480:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315497.692289:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1473 00000100:00100000:2.0:1761315497.692307:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2186 00000100:00100000:2.0:1761315497.692311:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322540352:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315497.692316:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315497.692338:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049074e99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322540160:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (530us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.692371:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4449 00000100:00100000:1.0:1761315497.692374:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315497.692382:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322540544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.692388:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540480 00000100:00100000:1.0:1761315497.692412:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315497.692415:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4450 00000100:00100000:3.0:1761315497.692421:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005b1075d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322540224:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315497.692430:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315497.692441:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540096 00000100:00100000:3.0:1761315497.692457:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005b1075d8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322540224:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (411us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.692464:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4450 00000100:00100000:1.0:1761315497.692473:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540288 00000100:00100000:3.0:1761315497.692479:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540544 00000100:00100000:1.0:1761315497.692498:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2787 00000100:00100000:3.0:1761315497.692502:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4451 00000100:00100000:1.0:1761315497.692505:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322540096:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315497.692510:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e08cda8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322540480:12345-0@lo:400:kworker/u8:0.0 00000004:00000024:1.0:1761315497.692531:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315497.692545:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e08cda8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322540480:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (231us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.692551:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4451 00000100:00100000:3.0:1761315497.692565:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4452 00000100:00100000:1.0:1761315497.692569:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322540096:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (912us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.692572:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f2cd44f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322540544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315497.692580:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2787 00000100:00100000:1.0:1761315497.692598:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2788 00000100:00100000:1.0:1761315497.692602:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322540288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315497.692625:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322540160:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315497.692671:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2187 00000100:00100000:0.0:1761315497.692678:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b06baf4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322540416:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315497.692757:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315497.693725:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322540288:12345-0@lo:400:kworker/u8:0.0 Request processed in 1119us (1560us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315497.693744:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2788 00000100:00100000:3.0:1761315497.693794:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f2cd44f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322540544:12345-0@lo:400:kworker/u8:0.0 Request processed in 1222us (1383us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315497.693801:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b06baf4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322540416:12345-0@lo:8:kworker/u8:0.0 Request processed in 1124us (1688us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315497.693805:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4452 00000100:00100000:3.0:1761315497.693859:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322540544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.693882:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322540480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.693916:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322540288:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.693926:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322540224:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.693934:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322540608:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.693948:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322540096:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315497.693985:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540032 00000100:00100000:3.0:1761315497.694023:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1761315497.694028:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322540032:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315497.694072:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322540032:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (1830us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315497.694079:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1761315497.694108:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062377729 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322540032:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315497.694659:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315497.694664:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315497.694668:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322540352:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315497.694684:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315497.694686:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315497.694688:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322540416:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315497.695011:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322540352:12345-0@lo:8:kworker/u8:0.0 Request processed in 2698us (2950us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315497.695020:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2186 00000100:00100000:0.0:1761315497.695080:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2187 00010000:00010000:2.0:1761315498.716981:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315499.720084:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315499.720948:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315499.720980:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315499.721291:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551616 00000100:00100000:0.0:1761315499.721338:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1653 00000100:00100000:0.0:1761315499.721360:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307551616:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315499.721428:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307551616:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (1352us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315499.721436:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1653 00000100:00100000:1.0:1761315499.722049:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551744 00000100:00100000:1.0:1761315499.722097:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551680 00000100:00100000:1.0:1761315499.722117:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4453 00000100:00100000:1.0:1761315499.722122:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b4b1da9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307551744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315499.722160:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b4b1da9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307551744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 37us (1215us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315499.722167:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4453 00000100:00100000:1.0:1761315499.722179:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4454 00000100:00100000:1.0:1761315499.722183:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000083166c01 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307551680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315499.722201:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000083166c01 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307551680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 18us (1221us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315499.722206:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4454 00000100:00100000:2.0:1761315499.722673:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315499.722953:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551808 00000100:00100000:0.0:1761315499.722990:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2188 00000100:00100000:0.0:1761315499.722996:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009dafc9f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307551808:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315499.723099:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315499.723133:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009dafc9f4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307551808:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 136us (463us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315499.723140:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2188 00010000:00010000:1.0:1761315499.739548:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315500.251628:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322540672:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315500.251700:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315500.251805:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540672 00000100:00100000:2.0:1761315500.252450:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2789 00000100:00100000:2.0:1761315500.252457:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322540672:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315500.252493:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315500.253531:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322540672:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1070us (1829us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315500.253542:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2789 00000100:00100000:3.0:1761315500.254463:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322540672:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315500.763585:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315501.790652:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315502.812066:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315502.812085:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322540736:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315502.812154:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315502.812319:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540736 00000100:00100000:3.0:1761315502.812391:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2790 00000100:00100000:3.0:1761315502.812397:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322540736:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315502.812430:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315502.812468:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322540736:12345-0@lo:41:osp-pre-0-1.0 Request processed in 70us (315us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315502.812475:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2790 00000100:00100000:0.0:1761315502.812562:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322540736:0@lo:41:osp-pre-0-1.0 00010000:00010000:1.0:1761315502.812667:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315502.812774:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000613c8591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322540800:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315502.812805:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315502.812881:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540800 00000100:00100000:0.0:1761315502.812918:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 977 00000100:00100000:0.0:1761315502.812922:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322540800:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315502.812942:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315502.812975:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322540800:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (171us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315502.812981:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 977 00000100:00100000:0.0:1761315502.813009:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000613c8591 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322540800:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1761315502.813282:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315502.813287:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315502.813290:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315502.813297:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3745 00000100:00080000:2.0:1761315502.813299:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315502.813302:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315502.813320:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006f75a3eb x1846869322541120/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315502.813333:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315502.813336:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315502.813337:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315502.813340:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3745 00000100:00080000:2.0:1761315502.813366:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315502.813370:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315502.813381:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ab2dda9a x1846869322541184/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315502.813414:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ab60fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541056:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813442:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315502.813460:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761315502.813465:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d653401f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322540992:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813489:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315502.813502:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a82025fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813524:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315502.813549:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541376 00000100:00100000:2.0:1761315502.813597:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1474 00000100:00100000:2.0:1761315502.813602:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813638:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541376:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (114us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315502.813646:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1474 00000100:00100000:2.0:1761315502.813669:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541056 00000100:00100000:2.0:1761315502.813691:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2791 00000100:00100000:2.0:1761315502.813694:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813719:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541056:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (278us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315502.813724:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2791 00000100:00100000:2.0:1761315502.813750:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a82025fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.813757:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ab60fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315502.813825:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006f75a3eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322541120:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315502.813853:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315502.813878:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab2dda9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322541184:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315502.813900:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315502.813929:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540992 00000100:00100000:1.0:1761315502.813956:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4455 00000100:00100000:1.0:1761315502.813959:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036336669 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322540992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315502.813988:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036336669 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322540992:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (498us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315502.813993:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4455 00000100:00100000:1.0:1761315502.814014:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541120 00000100:00100000:1.0:1761315502.814042:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541184 00000100:00100000:1.0:1761315502.814057:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2189 00000100:00100000:1.0:1761315502.814060:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322541120:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315502.814139:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315502.814174:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322541120:12345-0@lo:8:kworker/u8:2.0 Request processed in 112us (321us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315502.814179:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2189 00000100:00100000:1.0:1761315502.814189:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2190 00000100:00100000:1.0:1761315502.814192:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322541184:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315502.814264:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315502.814287:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322541184:12345-0@lo:8:kworker/u8:2.0 Request processed in 94us (386us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315502.814292:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2190 00000100:00080000:1.0:1761315502.814323:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315502.814326:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315502.814329:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006f75a3eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322541120:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315502.814335:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315502.814338:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315502.814340:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ab2dda9a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322541184:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814492:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000028e3290 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322540928:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814522:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315502.814550:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322541312:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814571:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315502.814601:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315502.814607:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa5dc3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322540864:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814631:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315502.814644:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004131e225 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322541248:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814664:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315502.814687:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540928 00000100:00100000:3.0:1761315502.814712:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541312 00000100:00100000:3.0:1761315502.814727:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541248 00000100:00100000:3.0:1761315502.814742:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4456 00000100:00100000:3.0:1761315502.814746:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057d0d0b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322540928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814779:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057d0d0b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322540928:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (257us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315502.814785:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4456 00000100:00100000:3.0:1761315502.814798:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4457 00000100:00100000:3.0:1761315502.814801:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000375aab8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322541312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814825:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000375aab8f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322541312:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315502.814831:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4457 00000100:00100000:3.0:1761315502.814841:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4458 00000100:00100000:3.0:1761315502.814844:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009404a1e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322541248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814864:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009404a1e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322541248:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (200us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315502.814869:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4458 00000100:00100000:3.0:1761315502.814889:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322540864 00000100:00100000:3.0:1761315502.814905:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1654 00000100:00100000:3.0:1761315502.814908:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322540864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814931:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322540864:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315502.814937:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1654 00000100:00100000:3.0:1761315502.814968:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa5dc3b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322540864:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814976:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004131e225 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322541248:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814984:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000028e3290 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322540928:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315502.814990:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322541312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315502.815196:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d653401f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322540992:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761315503.835471:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315504.841817:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315504.841923:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315504.843162:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551872 00000100:00100000:0.0:1761315504.843177:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315504.843211:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1655 00000100:00100000:1.0:1761315504.843219:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307551872:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315504.843487:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552000 00000100:00100000:3.0:1761315504.843515:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307551936 00000100:00100000:3.0:1761315504.843529:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4459 00000100:00100000:3.0:1761315504.843533:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000613c8591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552000:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315504.843598:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4460 00000100:00100000:2.0:1761315504.843602:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e36ce75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307551936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315504.843696:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315504.843706:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307551872:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 488us (1897us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315504.843714:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1655 00000100:00100000:0.0:1761315504.843808:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552064 00000100:00100000:0.0:1761315504.843846:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2191 00000100:00100000:0.0:1761315504.843850:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc5ec675 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307552064:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:3.0:1761315504.843946:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000613c8591 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552000:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 411us (2023us total) trans 0 rc 0/0 00010000:02020000:0.0:1761315504.843950:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315504.843953:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4459 00000100:00100000:0.0:1761315504.843976:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc5ec675 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307552064:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 126us (282us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315504.843982:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2191 00000100:00100000:2.0:1761315504.844156:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e36ce75d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307551936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 552us (980us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315504.844162:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4460 00010000:00010000:2.0:1761315504.862177:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315505.373160:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541440:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315505.373237:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315505.374672:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541440 00000100:00100000:0.0:1761315505.374747:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2792 00000100:00100000:0.0:1761315505.374755:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541440:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315505.374791:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315505.374861:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322541440:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (1625us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315505.374878:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2792 00000100:00100000:2.0:1761315505.375107:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322541440:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315505.883705:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315506.907589:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315507.931634:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315507.931743:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322541504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315507.931809:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315507.931867:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541504 00000100:00100000:1.0:1761315507.931904:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 978 00000100:00100000:1.0:1761315507.931910:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322541504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315507.931937:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315507.931973:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322541504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (165us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315507.931980:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 978 00000100:00100000:1.0:1761315507.932018:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008d9901b3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322541504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315507.932366:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315507.932375:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322541568:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315507.932401:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315507.932475:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541568 00000100:00100000:3.0:1761315507.933330:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2793 00000100:00100000:3.0:1761315507.933337:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322541568:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315507.933403:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315507.934164:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322541568:12345-0@lo:41:osp-pre-0-1.0 Request processed in 825us (1760us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315507.934174:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2793 00000100:00100000:1.0:1761315507.934622:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322541568:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315507.935761:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315507.935767:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315507.935770:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315507.935776:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3750 00000100:00080000:0.0:1761315507.935778:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315507.935783:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315507.935803:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004cf2823d x1846869322541888/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315507.935815:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315507.935817:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315507.935818:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315507.935821:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3750 00000100:00080000:0.0:1761315507.935822:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315507.935825:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315507.935833:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007ebcc50d x1846869322541952/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315507.935939:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322541888:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315507.935995:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322541824:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315507.935998:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315507.936021:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322541952:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315507.936038:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936038:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936057:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315507.936061:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322541760:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315507.936073:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541888 00000100:00100000:2.0:1761315507.936082:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936096:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322542144:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315507.936099:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541952 00000100:00100000:2.0:1761315507.936111:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315507.936114:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2192 00000100:00100000:1.0:1761315507.936119:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322541888:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936133:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541760 00000100:00100000:2.0:1761315507.936157:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4461 00000100:00100000:2.0:1761315507.936175:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c5722f38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322541760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936215:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c5722f38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322541760:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (133us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315507.936222:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4461 00000100:00100000:2.0:1761315507.936244:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542144 00000100:00100000:2.0:1761315507.936266:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1761315507.936269:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322542144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936296:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322542144:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315507.936301:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1475 00000100:00100000:2.0:1761315507.936311:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2193 00000100:00100000:2.0:1761315507.936313:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322541952:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315507.936418:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315507.936449:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322541952:12345-0@lo:8:kworker/u8:0.0 Request processed in 134us (411us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315507.936455:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2193 00000100:00080000:2.0:1761315507.936490:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315507.936494:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315507.936497:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322541952:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936521:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322541760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936529:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322542144:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936579:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322541632:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936607:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936622:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322542016:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936641:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936658:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315507.936663:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322541696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936682:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936693:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322542080:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936711:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315507.936733:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542016 00000100:00100000:2.0:1761315507.936752:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541696 00000100:00100000:2.0:1761315507.936765:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542080 00000100:00100000:2.0:1761315507.936782:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4462 00000100:00100000:2.0:1761315507.936786:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038a955f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322542016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315507.936816:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038a955f1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322542016:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (175us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315507.936821:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4462 00000100:00100000:2.0:1761315507.936834:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4463 00000100:00100000:2.0:1761315507.936838:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322541696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315507.936889:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541632 00000100:00100000:0.0:1761315507.936918:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656 00000100:00100000:0.0:1761315507.936922:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322541632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315507.936959:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322541632:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (351us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315507.936965:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656 00000100:00100000:0.0:1761315507.936991:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4464 00000100:00100000:0.0:1761315507.936994:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322542080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315507.937021:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088562e31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322542080:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (310us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315507.937027:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4464 00000100:00100000:3.0:1761315507.937201:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322541824 00000100:00100000:3.0:1761315507.937245:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2794 00000100:00100000:3.0:1761315507.937250:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322541824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315507.937293:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322541824:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (1255us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315507.937300:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2794 00000100:00100000:3.0:1761315507.937332:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322541824:0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315507.937376:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315507.937407:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322542080:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315507.937422:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322541632:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315507.937427:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322541888:12345-0@lo:8:kworker/u8:0.0 Request processed in 1307us (1429us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315507.937436:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322542016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315507.937437:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2192 00000100:00080000:1.0:1761315507.937472:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315507.937476:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315507.937480:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322541888:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315507.937656:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1cce0f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322541696:12345-0@lo:400:kworker/u8:0.0 Request processed in 816us (972us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315507.937663:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4463 00000100:00100000:3.0:1761315507.937703:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322541696:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315508.955670:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315509.963381:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315509.963423:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315509.963461:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315509.963489:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552320 00000100:00100000:0.0:1761315509.963514:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315509.963550:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2194 00000100:00100000:1.0:1761315509.963557:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307552320:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315509.963602:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552128 00000100:00100000:2.0:1761315509.963637:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1657 00000100:00100000:2.0:1761315509.963641:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307552128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00010000:02020000:1.0:1761315509.963652:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315509.963707:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307552128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 64us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315509.963717:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1657 00000100:00100000:3.0:1761315509.964049:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552256 00000100:00100000:3.0:1761315509.964088:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552192 00000100:00100000:3.0:1761315509.964118:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4465 00000100:00100000:3.0:1761315509.964122:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307552256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315509.964162:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fed2a801 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307552256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 39us (702us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315509.964170:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4465 00000100:00100000:3.0:1761315509.964184:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4466 00000100:00100000:3.0:1761315509.964188:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307552192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315509.964277:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b68078e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307552192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 86us (762us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315509.964284:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4466 00010000:00010000:2.0:1761315509.979699:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315509.982784:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307552320:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 19219us (19435us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315509.982801:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2194 00000100:00100000:3.0:1761315510.492116:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322542208:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315510.492188:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315510.492904:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542208 00000100:00100000:2.0:1761315510.492962:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2795 00000100:00100000:2.0:1761315510.492968:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322542208:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315510.493003:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315510.493063:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322542208:12345-0@lo:41:osp-pre-1-0.0 Request processed in 93us (875us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315510.493069:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2795 00000100:00100000:0.0:1761315510.493277:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322542208:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315511.009052:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315512.029337:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315513.051521:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315513.051556:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322542400:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.051567:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d164467 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322542464:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315513.051568:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315513.051577:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315513.051580:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315513.051586:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3755 00000100:00080000:2.0:1761315513.051589:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315513.051595:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322542272:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1761315513.051596:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315513.051629:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001883266f x1846869322542656/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315513.051635:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315513.051635:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315513.051661:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315513.051664:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315513.051666:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315513.051669:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3755 00000100:00080000:2.0:1761315513.051671:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315513.051674:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315513.051684:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000137dd527 x1846869322542720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315513.051732:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542400 00000100:00100000:2.0:1761315513.051771:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4467 00000100:00100000:2.0:1761315513.051777:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005abee3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322542400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.051981:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322542848:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.052006:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315513.052029:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315513.052035:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322542784:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.052062:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542848 00000100:00100000:1.0:1761315513.052065:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315513.052071:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005abee3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322542400:12345-0@lo:400:kworker/u8:0.0 Request processed in 291us (441us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315513.052080:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4467 00000100:00100000:0.0:1761315513.052093:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542784 00000100:00100000:1.0:1761315513.052102:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322542656:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315513.052103:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4468 00000100:00100000:2.0:1761315513.052112:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322542848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.052134:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315513.052158:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322542720:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315513.052162:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5601a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322542848:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (153us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315513.052184:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315513.052213:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4468 00000100:00100000:1.0:1761315513.052216:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542656 00000100:00100000:0.0:1761315513.052230:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4469 00000100:00100000:0.0:1761315513.052236:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322542784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.052243:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542720 00000100:00100000:2.0:1761315513.052269:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7db22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322542848:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.052270:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2195 00000100:00100000:0.0:1761315513.052276:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000741fc930 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322542784:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (212us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315513.052279:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322542656:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315513.052288:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4469 00000100:00100000:2.0:1761315513.052300:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322542784:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.052335:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2196 00000100:00100000:0.0:1761315513.052340:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322542720:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315513.052408:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a83abb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322542336:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315513.052441:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315513.052443:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315513.052461:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f48f386 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322542528:0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761315513.052487:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315513.052487:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322542656:12345-0@lo:8:kworker/u8:0.0 Request processed in 208us (352us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315513.052495:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315513.052511:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2195 00000100:00080000:1.0:1761315513.052539:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315513.052543:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315513.052546:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322542656:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315513.052576:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322543040:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315513.052996:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315513.053027:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322542912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.053037:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322542720:12345-0@lo:8:kworker/u8:0.0 Request processed in 695us (853us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315513.053045:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2196 00000100:00100000:2.0:1761315513.053057:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315513.053071:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542272 00000100:00100000:2.0:1761315513.053086:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315513.053093:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039d27018 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322542976:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.053102:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 979 00000100:00100000:2.0:1761315513.053115:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315513.053143:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542336 00000100:00100000:2.0:1761315513.053166:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1658 00000100:00100000:2.0:1761315513.053170:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322542336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315513.053202:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322542336:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (767us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315513.053208:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1658 00000100:00100000:0.0:1761315513.053378:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322542272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315513.053401:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315513.053407:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315513.053445:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322542272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (450us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315513.053446:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322542400:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.053457:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 979 00000100:00100000:1.0:1761315513.053482:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542464 00000100:00100000:0.0:1761315513.053487:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542528 00000100:00100000:1.0:1761315513.053524:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543040 00000100:00100000:3.0:1761315513.053538:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322542272:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315513.053546:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4470 00000100:00100000:3.0:1761315513.053549:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322542592:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.053552:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2796 00000100:00100000:0.0:1761315513.053555:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7db22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322542528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.053561:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322542464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315513.053585:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315513.053605:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542592 00000100:00100000:0.0:1761315513.053615:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7db22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322542528:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (1122us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315513.053627:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4470 00000100:00100000:1.0:1761315513.053627:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322542464:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (1995us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315513.053667:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a83abb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322542336:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315513.053674:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315513.053679:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315513.053683:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322542720:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315513.053686:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f48f386 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322542528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315513.053689:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2797 00000100:00100000:1.0:1761315513.053699:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2796 00000100:00100000:2.0:1761315513.053700:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322543040:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315513.053712:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542912 00000100:00100000:1.0:1761315513.053724:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2798 00000004:00000024:2.0:1761315513.053727:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315513.053731:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322542592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.053747:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322542976 00000100:00100000:3.0:1761315513.053758:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1476 00000100:00100000:3.0:1761315513.053765:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322542912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315513.053773:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322543040:12345-0@lo:41:osp-pre-0-1.0 Request processed in 74us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315513.053780:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1477 00000100:00100000:2.0:1761315513.053783:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2797 00000100:00100000:1.0:1761315513.053787:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322542592:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (201us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315513.053788:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322542976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.053802:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2798 00000100:00100000:3.0:1761315513.053812:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322542912:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (754us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315513.053819:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1476 00000100:00100000:2.0:1761315513.053835:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d26c929f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322542912:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.053849:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d164467 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322542464:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315513.053856:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322542976:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (739us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315513.053864:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1477 00000100:00100000:2.0:1761315513.053867:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039d27018 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322542976:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315513.053889:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322542592:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315513.053895:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322543040:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315514.076116:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315515.082529:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315515.082573:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315515.082652:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315515.082657:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552448 00000100:00100000:3.0:1761315515.082689:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552384 00000100:00100000:0.0:1761315515.082711:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552512 00000100:00100000:0.0:1761315515.082737:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4471 00000100:00100000:0.0:1761315515.082743:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307552448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315515.082769:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4472 00000100:00100000:1.0:1761315515.082776:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d26c929f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315515.082818:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039d27018 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307552448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 73us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315515.082820:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d26c929f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 42us (167us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315515.082832:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4471 00000100:00100000:1.0:1761315515.082839:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4472 00000100:00100000:3.0:1761315515.083002:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1659 00000100:00100000:3.0:1761315515.083008:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307552384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315515.083053:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307552384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 45us (533us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315515.083060:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1659 00000100:00100000:2.0:1761315515.085608:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315515.085712:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552576 00000100:00100000:1.0:1761315515.085771:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2197 00000100:00100000:1.0:1761315515.085778:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307552576:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315515.085904:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315515.085979:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307552576:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 201us (377us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315515.085987:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2197 00010000:00010000:1.0:1761315515.099924:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315515.613837:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322543104:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315515.613909:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315515.613972:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543104 00000100:00100000:1.0:1761315515.614029:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2799 00000100:00100000:1.0:1761315515.614035:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322543104:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315515.614069:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315515.614643:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322543104:12345-0@lo:41:osp-pre-1-0.0 Request processed in 606us (735us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315515.614652:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2799 00000100:00100000:0.0:1761315515.614736:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322543104:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315516.123521:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315517.148466:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315518.175822:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315518.175845:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322543168:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315518.175928:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315518.175934:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315518.175937:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315518.175942:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3760 00000100:00080000:0.0:1761315518.175945:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315518.175951:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315518.175969:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000011fc2a84 x1846869322543552/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315518.175985:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315518.175987:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315518.175989:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315518.175992:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3760 00000100:00080000:0.0:1761315518.175994:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315518.175996:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315518.176006:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f72ffb37 x1846869322543616/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315518.176080:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322543360:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176138:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176166:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322543744:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176189:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176207:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315518.176212:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322543296:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176231:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176243:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322543680:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176262:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176284:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322543552:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176321:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315518.176335:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176338:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322543616:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315518.176389:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322543232:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176394:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315518.176429:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315518.176430:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176449:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543360 00000100:00100000:3.0:1761315518.176473:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322543488:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.176501:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176504:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2800 00000100:00100000:0.0:1761315518.176510:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322543360:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.176519:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322543872:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.176540:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315518.176559:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315518.176565:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322543424:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176571:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322543360:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (432us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176579:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2800 00000100:00100000:3.0:1761315518.176590:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176597:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543488 00000100:00100000:3.0:1761315518.176606:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322543808:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176625:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2801 00000100:00100000:3.0:1761315518.176629:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315518.176633:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322543488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.176658:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543168 00000100:00100000:0.0:1761315518.176675:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322543488:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176681:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2801 00000100:00100000:0.0:1761315518.176702:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543744 00000100:00100000:0.0:1761315518.176724:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543296 00000100:00100000:0.0:1761315518.176738:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543680 00000100:00100000:0.0:1761315518.176748:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543424 00000100:00100000:0.0:1761315518.176758:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4473 00000100:00100000:0.0:1761315518.176761:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322543744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176793:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d9901b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322543744:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (603us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176800:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4473 00000100:00100000:3.0:1761315518.176804:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 980 00000100:00100000:3.0:1761315518.176811:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322543168:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315518.176816:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4474 00000100:00100000:0.0:1761315518.176821:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0629bf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322543296:12345-0@lo:400:kworker/u8:2.0 00002000:00000024:3.0:1761315518.176834:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315518.176848:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0629bf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322543296:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (615us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176854:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4474 00000100:00100000:0.0:1761315518.176864:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4475 00000100:00100000:0.0:1761315518.176867:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f668161 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322543680:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176889:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f668161 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322543680:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (628us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176894:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4475 00000100:00100000:0.0:1761315518.176902:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4476 00000100:00100000:0.0:1761315518.176905:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322543424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.176922:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c899dd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322543424:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (335us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.176927:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4476 00000100:00100000:0.0:1761315518.176948:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543552 00000100:00100000:0.0:1761315518.176960:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543616 00000100:00100000:0.0:1761315518.176968:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543872 00000100:00100000:0.0:1761315518.176978:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543808 00000100:00100000:0.0:1761315518.176985:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1478 00000100:00100000:0.0:1761315518.176987:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322543872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.177009:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322543872:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (469us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.177014:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1478 00000100:00100000:0.0:1761315518.177021:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1479 00000100:00100000:0.0:1761315518.177024:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322543808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.177042:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322543808:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.177047:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1479 00000100:00100000:0.0:1761315518.177054:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2198 00000100:00100000:0.0:1761315518.177057:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322543552:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177088:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322543168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 277us (755us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315518.177096:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 980 00000100:00100000:3.0:1761315518.177119:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543232 00010000:02020000:0.0:1761315518.177141:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315518.177142:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1660 00000100:00100000:3.0:1761315518.177148:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322543232:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.177175:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322543552:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (853us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315518.177181:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2198 00000100:00100000:3.0:1761315518.177185:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322543232:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (756us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315518.177195:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2199 00000100:00100000:3.0:1761315518.177195:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1660 00000100:00100000:0.0:1761315518.177203:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322543616:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315518.177277:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315518.177287:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322543296:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177303:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322543680:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315518.177305:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322543616:12345-0@lo:8:kworker/u8:2.0 Request processed in 101us (913us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315518.177314:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2199 00000100:00100000:3.0:1761315518.177319:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322543360:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177331:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322543744:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315518.177368:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315518.177373:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315518.177377:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322543552:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315518.177389:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315518.177391:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315518.177394:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f72ffb37 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322543616:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177412:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c6ba408 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322543424:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177425:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322543808:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177433:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065a64462 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322543232:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177441:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf2823d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322543488:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177460:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038cef707 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322543872:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315518.177484:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322543168:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315518.177728:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322543936:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315518.177761:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315518.177834:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322543936 00000100:00100000:2.0:1761315518.177876:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2802 00000100:00100000:2.0:1761315518.177880:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34014aa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322543936:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315518.177904:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315518.177947:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34014aa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322543936:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315518.177953:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2802 00000100:00100000:0.0:1761315518.178072:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322543936:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315519.195682:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315520.204386:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315520.204436:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315520.204601:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552704 00000100:00100000:1.0:1761315520.204607:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315520.204631:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315520.204646:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552768 00000100:00100000:2.0:1761315520.204667:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4477 00000100:00100000:2.0:1761315520.204673:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307552704:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315520.204741:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091a6e538 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307552704:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (304us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315520.204748:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4477 00000100:00100000:2.0:1761315520.204765:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4478 00000100:00100000:2.0:1761315520.204769:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552768:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315520.204794:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f2f775a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307552768:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315520.204800:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4478 00000100:00100000:1.0:1761315520.205110:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552832 00000100:00100000:1.0:1761315520.205141:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2200 00000100:00100000:1.0:1761315520.205145:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307552832:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315520.205245:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315520.205292:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307552832:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 145us (660us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315520.205299:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2200 00000100:00100000:3.0:1761315520.205608:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552640 00000100:00100000:3.0:1761315520.205681:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1661 00000100:00100000:3.0:1761315520.205686:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307552640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315520.205729:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307552640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (1351us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315520.205735:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1661 00010000:00010000:0.0:1761315520.220698:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315520.731563:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322544000:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315520.731626:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315520.731726:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544000 00000100:00100000:2.0:1761315520.732576:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2803 00000100:00100000:2.0:1761315520.732584:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322544000:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315520.732622:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315520.733670:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322544000:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1083us (2041us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315520.733680:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2803 00000100:00100000:3.0:1761315520.733850:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322544000:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315521.243541:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315522.267989:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315523.291628:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322544064:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315523.291692:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315523.291726:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315523.291733:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322544128:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315523.291738:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544064 00000100:00100000:3.0:1761315523.291761:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315523.291788:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 981 00000100:00100000:2.0:1761315523.291795:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322544064:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315523.291808:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544128 00002000:00000024:2.0:1761315523.291823:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315523.291840:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2804 00000100:00100000:3.0:1761315523.291845:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9965e9b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322544128:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315523.291871:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315523.291882:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322544064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (190us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315523.291890:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 981 00000100:00100000:3.0:1761315523.291913:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9965e9b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322544128:12345-0@lo:41:osp-pre-0-1.0 Request processed in 65us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315523.291921:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2804 00000100:00100000:2.0:1761315523.291932:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009edc9989 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322544128:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315523.291950:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322544064:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315523.292093:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315523.292360:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322544192:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315523.292397:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315523.292984:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322544256:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315523.293019:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315523.293106:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544192 00000100:00100000:3.0:1761315523.293142:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662 00000100:00100000:3.0:1761315523.293146:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322544192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.293186:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322544192:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (788us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315523.293192:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662 00000100:00100000:0.0:1761315523.293384:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322544192:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315523.293409:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544256 00000100:00100000:0.0:1761315523.293454:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4479 00000100:00100000:0.0:1761315523.293459:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009edc9989 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322544256:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315523.293544:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315523.293548:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315523.293552:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315523.293557:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3765 00000100:00080000:1.0:1761315523.293559:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315523.293563:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315523.293582:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b7b15fb6 x1846869322544448/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315523.293595:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315523.293598:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315523.293599:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315523.293602:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3765 00000100:00080000:1.0:1761315523.293603:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315523.293605:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315523.293614:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a9137a0c x1846869322544512/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315523.294113:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009edc9989 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322544256:12345-0@lo:400:kworker/u8:0.0 Request processed in 651us (1091us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315523.294118:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322544384:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315523.294124:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4479 00000100:00100000:0.0:1761315523.294150:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097cfe154 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322544576:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294156:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315523.294179:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315523.294185:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315523.294187:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322544320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294216:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315523.294218:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322544448:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294249:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544576 00000100:00100000:0.0:1761315523.294252:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315523.294272:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322544512:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294278:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544320 00000100:00100000:0.0:1761315523.294300:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315523.294303:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4480 00000100:00100000:3.0:1761315523.294308:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000200917a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322544576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315523.294333:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544384 00000100:00100000:3.0:1761315523.294371:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000200917a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322544576:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (187us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315523.294377:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4480 00000100:00100000:0.0:1761315523.294386:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2805 00000100:00100000:0.0:1761315523.294392:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322544384:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294396:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4481 00000100:00100000:3.0:1761315523.294402:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038cef707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322544320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315523.294449:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322544384:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315523.294455:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038cef707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322544320:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (238us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315523.294460:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2805 00000100:00100000:3.0:1761315523.294467:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4481 00000100:00100000:0.0:1761315523.294483:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544448 00000100:00100000:0.0:1761315523.294504:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544512 00000100:00100000:0.0:1761315523.294523:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2201 00000100:00100000:0.0:1761315523.294527:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322544448:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315523.294531:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2202 00000100:00100000:3.0:1761315523.294537:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322544512:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315523.294944:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761315523.294957:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315523.294988:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322544448:12345-0@lo:8:kworker/u8:0.0 Request processed in 459us (737us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315523.294995:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322544512:12345-0@lo:8:kworker/u8:0.0 Request processed in 456us (696us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315523.294998:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2201 00000100:00100000:3.0:1761315523.295008:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2202 00000100:00100000:0.0:1761315523.295044:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097cfe154 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322544576:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.295054:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322544320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315523.295067:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322544384:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315523.295083:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315523.295089:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315523.295093:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322544448:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315523.295103:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315523.295106:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315523.295109:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322544512:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315523.295212:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322544256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315523.295448:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315523.295457:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000698107b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322544704:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315523.295487:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315523.295512:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544704 00000100:00100000:1.0:1761315523.295539:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1480 00000100:00100000:1.0:1761315523.295543:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000249f09c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322544704:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315523.295578:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000249f09c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322544704:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (91us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315523.295583:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1480 00000100:00100000:1.0:1761315523.295609:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000698107b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322544704:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315523.296313:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322544640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315523.296374:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315523.296713:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544640 00000100:00100000:1.0:1761315523.296756:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4482 00000100:00100000:1.0:1761315523.296761:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d164467 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322544640:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315523.296805:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d164467 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322544640:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315523.296812:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4482 00000100:00100000:2.0:1761315523.296892:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322544640:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315524.315496:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315525.322477:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315525.322528:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315525.322560:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315525.322636:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552960 00000100:00100000:2.0:1761315525.322666:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315525.322686:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553024 00000100:00100000:1.0:1761315525.322710:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4483 00000100:00100000:1.0:1761315525.322716:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f48f386 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307552960:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315525.322716:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553088 00000100:00100000:2.0:1761315525.322745:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2203 00000100:00100000:2.0:1761315525.322750:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307553088:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:1.0:1761315525.322789:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f48f386 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307552960:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 72us (320us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315525.322797:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4483 00000100:00100000:1.0:1761315525.322813:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4484 00000100:00100000:1.0:1761315525.322816:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307553024:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315525.322837:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a83abb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307553024:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 20us (279us total) trans 0 rc 0/0 00010000:02020000:2.0:1761315525.322843:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315525.322845:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4484 00000100:00100000:2.0:1761315525.322879:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307553088:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 128us (215us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315525.322887:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2203 00000100:00100000:0.0:1761315525.323479:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307552896 00000100:00100000:0.0:1761315525.323524:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1663 00000100:00100000:0.0:1761315525.323529:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307552896:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315525.323581:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307552896:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 50us (1053us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315525.323587:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1663 00010000:00010000:0.0:1761315525.340671:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315525.851640:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315525.851747:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322544768:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315525.851809:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315525.851869:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544768 00000100:00100000:1.0:1761315525.851929:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2806 00000100:00100000:1.0:1761315525.851936:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322544768:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315525.851968:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315525.852665:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322544768:12345-0@lo:41:osp-pre-1-0.0 Request processed in 727us (854us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315525.852674:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2806 00000100:00100000:3.0:1761315525.852743:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322544768:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315526.363551:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315527.388869:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315528.411537:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315528.411653:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315528.411658:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315528.411661:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315528.411670:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3770 00000100:00080000:0.0:1761315528.411673:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315528.411679:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315528.411698:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bbb442e6 x1846869322545152/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315528.411707:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322545024:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315528.411714:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315528.411717:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315528.411719:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315528.411722:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3770 00000100:00080000:0.0:1761315528.411725:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315528.411727:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315528.411737:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f159976f x1846869322545216/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315528.411777:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.411783:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322545152:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315528.411814:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000849700d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322545408:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315528.411826:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.411846:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f159976f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322545216:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315528.411851:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.411876:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315528.411876:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315528.411884:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d911da4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322544960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315528.411912:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.411914:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545152 00000100:00100000:1.0:1761315528.411933:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322545344:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315528.411944:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545408 00000100:00100000:1.0:1761315528.411960:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.411967:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545216 00000100:00100000:0.0:1761315528.411982:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1481 00000100:00100000:0.0:1761315528.411987:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322545408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315528.412009:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2204 00000100:00100000:1.0:1761315528.412015:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322545152:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315528.412034:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322545408:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (184us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315528.412040:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1481 00000100:00100000:0.0:1761315528.412055:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2205 00000100:00100000:0.0:1761315528.412058:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322545216:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315528.412195:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:0.0:1761315528.412229:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315528.412242:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322545152:12345-0@lo:8:kworker/u8:2.0 Request processed in 225us (415us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315528.412250:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2204 00000100:00100000:0.0:1761315528.412260:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322545216:12345-0@lo:8:kworker/u8:2.0 Request processed in 199us (384us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315528.412267:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2205 00000100:00100000:1.0:1761315528.412277:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544960 00000100:00100000:0.0:1761315528.412308:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545024 00000100:00100000:1.0:1761315528.412311:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2807 00000100:00100000:1.0:1761315528.412316:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322544960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315528.412337:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545344 00000100:00100000:1.0:1761315528.412373:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322544960:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315528.412381:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2807 00000100:00100000:0.0:1761315528.412384:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4485 00000100:00100000:0.0:1761315528.412389:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000698107b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322545024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315528.412422:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4486 00000100:00100000:1.0:1761315528.412428:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088265ecb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322545344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315528.412433:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000698107b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322545024:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (658us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315528.412445:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4485 00000100:00100000:1.0:1761315528.412464:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088265ecb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322545344:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (505us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315528.412471:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4486 00000100:00100000:0.0:1761315528.412492:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d911da4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322544960:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315528.412507:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761315528.412510:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322545344:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315528.412512:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315528.412520:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322545152:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315528.412526:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000849700d4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322545408:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315528.412533:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315528.412537:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315528.412542:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322545024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315528.412542:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f159976f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322545216:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315528.412720:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322544832:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.412752:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315528.412900:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000932fbacb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322545088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.412923:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315528.412937:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322545472:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.412954:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315528.412969:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315528.412975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322544896:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.412993:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315528.413005:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e962613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322545280:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413022:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315528.413044:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545472 00000100:00100000:3.0:1761315528.413110:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1482 00000100:00100000:3.0:1761315528.413115:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322545472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413149:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322545472:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (195us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315528.413155:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1482 00000100:00100000:3.0:1761315528.413174:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545088 00000100:00100000:3.0:1761315528.413200:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2808 00000100:00100000:3.0:1761315528.413204:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322545088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413231:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322545088:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (309us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315528.413237:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2808 00000100:00100000:3.0:1761315528.413257:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544896 00000100:00100000:3.0:1761315528.413274:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545280 00000100:00100000:3.0:1761315528.413287:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4487 00000100:00100000:3.0:1761315528.413290:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322544896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413318:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c6ba408 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322544896:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (325us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315528.413323:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4487 00000100:00100000:3.0:1761315528.413332:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4488 00000100:00100000:3.0:1761315528.413334:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322545280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413378:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f72ffb37 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322545280:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315528.413383:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4488 00000100:00100000:3.0:1761315528.413420:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322544896:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413428:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e962613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322545280:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413437:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000932fbacb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322545088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315528.413443:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322545472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315528.414116:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322545536:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315528.414150:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315528.414893:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322544832 00000100:00100000:0.0:1761315528.415222:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545536 00000100:00100000:0.0:1761315528.415246:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2809 00000100:00100000:0.0:1761315528.415250:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322545536:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315528.415276:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315528.415320:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322545536:12345-0@lo:41:osp-pre-0-1.0 Request processed in 69us (1170us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315528.415325:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2809 00000100:00100000:0.0:1761315528.415375:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002b7a700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322545536:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315528.415428:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315528.415435:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322545600:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315528.415460:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315528.415482:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545600 00000100:00100000:0.0:1761315528.415503:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 982 00000100:00100000:0.0:1761315528.415507:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322545600:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315528.415525:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315528.415554:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322545600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (93us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315528.415559:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 982 00000100:00100000:0.0:1761315528.415583:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322545600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315528.416386:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1664 00000100:00100000:2.0:1761315528.416406:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322544832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315528.416452:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322544832:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (3700us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315528.416458:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1664 00000100:00100000:0.0:1761315528.416545:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322544832:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315529.435614:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315530.440941:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315530.440990:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315530.441019:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315530.441055:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315530.441405:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553152 00000100:00100000:3.0:1761315530.441444:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1665 00000100:00100000:3.0:1761315530.441450:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307553152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315530.441505:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307553152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 55us (574us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315530.441511:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1665 00000100:00100000:0.0:1761315530.442493:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553280 00000100:00100000:0.0:1761315530.442539:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553216 00000100:00100000:0.0:1761315530.442566:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4489 00000100:00100000:0.0:1761315530.442571:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307553280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315530.442626:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e869ff13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307553280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 53us (1636us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315530.442632:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4489 00000100:00100000:0.0:1761315530.442646:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4490 00000100:00100000:0.0:1761315530.442649:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002b7a700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315530.442672:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002b7a700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 22us (1653us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315530.442679:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4490 00000100:00100000:0.0:1761315530.442700:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553344 00000100:00100000:0.0:1761315530.442718:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2206 00000100:00100000:0.0:1761315530.442722:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307553344:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315530.442809:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315530.442848:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307553344:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 125us (1793us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315530.442855:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2206 00010000:00010000:2.0:1761315530.461209:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315530.971671:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315530.971686:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322545664:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315530.971753:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315530.971800:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545664 00000100:00100000:1.0:1761315530.971845:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1761315530.971851:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322545664:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315530.971884:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315530.971923:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322545664:12345-0@lo:41:osp-pre-1-0.0 Request processed in 72us (171us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315530.971942:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2810 00000100:00100000:1.0:1761315530.971986:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322545664:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315531.484474:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315532.508452:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315533.536435:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315533.536453:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322545728:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315533.536514:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.536567:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545728 00000100:00100000:2.0:1761315533.536611:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2811 00000100:00100000:2.0:1761315533.536617:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322545728:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315533.536648:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315533.536699:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322545728:12345-0@lo:41:osp-pre-0-1.0 Request processed in 82us (186us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.536706:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2811 00000100:00100000:0.0:1761315533.536718:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322545728:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315533.539501:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a597521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322545792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.539569:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.539618:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545792 00000100:00100000:2.0:1761315533.539656:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1666 00000100:00100000:2.0:1761315533.539661:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322545792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.539707:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322545792:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (137us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.539714:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1666 00000100:00100000:2.0:1761315533.539756:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a597521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322545792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.541158:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322545856:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315533.541208:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315533.551952:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545856 00000100:00100000:2.0:1761315533.551988:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315533.551999:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5e3a129 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322545920:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315533.552000:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 983 00000100:00100000:0.0:1761315533.552006:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322545856:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315533.552034:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315533.552050:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315533.552069:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322545856:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (10862us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.552075:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545920 00000100:00100000:0.0:1761315533.552077:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 983 00000100:00100000:2.0:1761315533.552099:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4491 00000100:00100000:2.0:1761315533.552102:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b06baf4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322545920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.552150:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b06baf4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322545920:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (100us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.552156:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4491 00000100:00100000:2.0:1761315533.552191:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322545856:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315533.552212:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5e3a129 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322545920:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315533.556050:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315533.556056:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315533.556060:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315533.556067:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3775 00000100:00080000:1.0:1761315533.556070:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315533.556076:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315533.556095:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f2e87ad x1846869322546112/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315533.556113:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315533.556116:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315533.556118:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315533.556121:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3775 00000100:00080000:1.0:1761315533.556123:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315533.556126:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315533.556142:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f2e87ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322546112:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315533.556197:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315533.556254:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000093b81021 x1846869322546176/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315533.556272:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093b81021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322546176:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315533.556293:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315533.556405:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546112 00000100:00100000:1.0:1761315533.556433:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546176 00000100:00100000:1.0:1761315533.556447:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2207 00000100:00100000:1.0:1761315533.556452:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322546112:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315533.556541:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315533.556578:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322546112:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (381us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315533.556584:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2207 00000100:00100000:1.0:1761315533.556598:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2208 00000100:00100000:1.0:1761315533.556601:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322546176:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315533.556685:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315533.556706:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322546176:12345-0@lo:8:kworker/u8:0.0 Request processed in 104us (414us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315533.556712:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2208 00000100:00080000:1.0:1761315533.556739:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315533.556743:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315533.556747:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093b81021 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322546176:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315533.556755:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315533.556757:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315533.556759:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f2e87ad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322546112:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315533.556890:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a919a5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322545984:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.556922:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.556953:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5e3a129 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546368:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.556974:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.556991:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315533.556996:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009400013f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546048:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557017:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.557043:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546048 00000100:00100000:2.0:1761315533.557070:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2812 00000100:00100000:2.0:1761315533.557075:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546048:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557114:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546048:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.557120:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2812 00000100:00100000:2.0:1761315533.557138:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546368 00000100:00100000:2.0:1761315533.557159:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1483 00000100:00100000:2.0:1761315533.557163:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d653401f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557190:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d653401f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546368:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (216us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.557195:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1483 00000100:00100000:2.0:1761315533.557222:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009400013f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546048:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557231:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5e3a129 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546368:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557251:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322546240:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557273:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.557294:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322545984 00000100:00100000:2.0:1761315533.557315:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546240 00000100:00100000:2.0:1761315533.557332:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4492 00000100:00100000:2.0:1761315533.557335:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009dafc9f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322545984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557407:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009dafc9f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322545984:12345-0@lo:400:kworker/u8:0.0 Request processed in 71us (485us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.557413:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4492 00000100:00100000:2.0:1761315533.557424:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4493 00000100:00100000:2.0:1761315533.557427:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000028e3290 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322546240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557449:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000028e3290 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322546240:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (176us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.557454:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4493 00000100:00100000:2.0:1761315533.557475:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a919a5d1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322545984:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557497:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322546240:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557503:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315533.557508:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322546304:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557527:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315533.557544:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546304 00000100:00100000:2.0:1761315533.557562:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4494 00000100:00100000:2.0:1761315533.557565:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004131e225 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322546304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315533.557589:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004131e225 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322546304:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (61us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315533.557594:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4494 00000100:00100000:2.0:1761315533.557613:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322546304:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315534.561776:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315535.561957:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315535.561996:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315535.562027:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315535.562798:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553408 00000100:00100000:3.0:1761315535.562854:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1667 00000100:00100000:3.0:1761315535.562861:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315535.562933:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (982us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315535.562941:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1667 00000100:00100000:2.0:1761315535.563237:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315535.564371:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553472 00000100:00100000:1.0:1761315535.564432:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553536 00000100:00100000:1.0:1761315535.564460:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4495 00000100:00100000:1.0:1761315535.564465:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5dc3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315535.564530:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5dc3b5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 64us (2533us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315535.564537:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4495 00000100:00100000:1.0:1761315535.564553:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4496 00000100:00100000:1.0:1761315535.564556:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab2dda9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307553536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315535.564583:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab2dda9a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307553536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (2557us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315535.564589:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4496 00000100:00100000:3.0:1761315535.566019:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553600 00000100:00100000:3.0:1761315535.566826:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2209 00000100:00100000:3.0:1761315535.566833:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f75a3eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307553600:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315535.566931:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315535.567840:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f75a3eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307553600:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 1003us (4600us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315535.567852:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2209 00010000:00010000:2.0:1761315535.587687:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315536.091809:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315536.091831:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546432:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315536.091896:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315536.092256:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546432 00000100:00100000:3.0:1761315536.092301:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2813 00000100:00100000:3.0:1761315536.092308:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546432:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315536.092386:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315536.092440:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322546432:12345-0@lo:41:osp-pre-1-0.0 Request processed in 132us (545us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315536.092449:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2813 00000100:00100000:1.0:1761315536.092559:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322546432:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315536.607385:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315537.628662:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315538.652026:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315538.652048:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322546496:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315538.652111:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315538.652230:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315538.652243:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000010ab60fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322546560:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315538.652277:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315538.652337:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546560 00000100:00100000:1.0:1761315538.652424:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546496 00000100:00100000:1.0:1761315538.652479:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 984 00000100:00100000:1.0:1761315538.652486:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322546496:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315538.652490:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315538.652496:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315538.652500:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315538.652505:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3781 00000100:00080000:3.0:1761315538.652509:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315538.652514:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00002000:00000024:1.0:1761315538.652519:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315538.652534:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a9137a0c x1846869322546880/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315538.652552:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315538.652555:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315538.652556:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315538.652559:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3781 00000100:00080000:3.0:1761315538.652560:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315538.652564:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315538.652567:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322546496:12345-0@lo:13:osp-pre-1-1.0 Request processed in 79us (455us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315538.652577:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b7b15fb6 x1846869322546944/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315538.652578:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 984 00000100:00100000:1.0:1761315538.652611:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322546624:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315538.652627:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322546752:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.652629:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322546816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.652647:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315538.652673:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322547008:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315538.652677:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315538.652681:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315538.652705:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653275:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315538.653291:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322546496:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315538.653302:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315538.653310:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322546688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653337:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546816 00000100:00100000:1.0:1761315538.653369:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315538.653388:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322547072:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653393:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2814 00000100:00100000:3.0:1761315538.653398:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d0d8c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322546816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653412:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315538.653443:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d0d8c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322546816:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (769us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315538.653450:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2814 00000100:00100000:1.0:1761315538.653452:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322546880:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653477:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322546816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653489:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315538.653512:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322546944:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653518:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546752 00000100:00100000:1.0:1761315538.653538:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315538.653545:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547008 00000100:00100000:3.0:1761315538.653566:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546688 00000100:00100000:1.0:1761315538.653575:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547072 00000100:00100000:3.0:1761315538.653583:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4497 00000100:00100000:3.0:1761315538.653587:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d911da4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322546752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653602:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4498 00000100:00100000:1.0:1761315538.653608:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f2e87ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322547008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653622:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d911da4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322546752:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (950us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315538.653629:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4497 00000100:00100000:3.0:1761315538.653643:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4499 00000100:00100000:3.0:1761315538.653648:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093b81021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322546688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653654:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f2e87ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322547008:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (379us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315538.653662:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4498 00000100:00100000:1.0:1761315538.653680:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4500 00000100:00100000:3.0:1761315538.653684:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093b81021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322546688:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315538.653689:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a597521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322547072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.653696:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4499 00000100:00100000:0.0:1761315538.653699:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315538.653732:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a597521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322547072:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315538.653740:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322546752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315538.653742:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4500 00000100:00100000:3.0:1761315538.653744:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546880 00000100:00100000:3.0:1761315538.653768:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546944 00000100:00100000:1.0:1761315538.653783:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322546624 00000100:00100000:1.0:1761315538.653816:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668 00000100:00100000:1.0:1761315538.653820:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322546624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315538.653850:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2815 00000100:00100000:2.0:1761315538.653856:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322546560:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315538.653868:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322546624:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (1222us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315538.653875:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668 00000004:00000024:2.0:1761315538.653880:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00010000:00010000:1.0:1761315538.653903:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315538.653939:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547136 00000100:00100000:3.0:1761315538.654098:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1484 00000100:00100000:3.0:1761315538.654102:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654137:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547136:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (442us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315538.654142:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1484 00000100:00100000:0.0:1761315538.654153:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2210 00000100:00100000:0.0:1761315538.654163:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322546880:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315538.654163:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2211 00000100:00100000:1.0:1761315538.654172:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097cfe154 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322546944:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654213:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547136:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654226:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000010ab60fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322546560:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315538.654276:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322546688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654286:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322547072:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654293:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322546624:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315538.654300:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001dbe4ffe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322547008:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315538.654442:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322546560:12345-0@lo:41:osp-pre-0-1.0 Request processed in 66us (1645us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315538.654450:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2815 00010000:02020000:0.0:1761315538.654654:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315538.654655:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315538.654711:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322546880:12345-0@lo:8:kworker/u8:2.0 Request processed in 550us (1223us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315538.654715:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097cfe154 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322546944:12345-0@lo:8:kworker/u8:2.0 Request processed in 542us (1176us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315538.654722:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2210 00000100:00100000:1.0:1761315538.654728:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2211 00000100:00080000:2.0:1761315538.657252:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315538.657261:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315538.657267:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a9137a0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322546880:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315538.657277:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315538.657280:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315538.657282:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b7b15fb6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322546944:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315539.675625:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315540.680432:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315540.680495:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315540.680540:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315540.680651:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553728 00000100:00100000:0.0:1761315540.680733:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553792 00000100:00100000:0.0:1761315540.680758:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4501 00000100:00100000:0.0:1761315540.680767:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a82025fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307553728:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315540.680865:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a82025fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307553728:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 98us (447us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315540.680874:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4501 00000100:00100000:0.0:1761315540.680898:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4502 00000100:00100000:0.0:1761315540.680902:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010ab60fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307553792:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315540.680925:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010ab60fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307553792:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 23us (387us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315540.680931:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4502 00000100:00100000:1.0:1761315540.681081:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553664 00000100:00100000:1.0:1761315540.681124:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1669 00000100:00100000:1.0:1761315540.681128:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553664:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315540.681158:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553664:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 29us (663us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315540.681165:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1669 00000100:00100000:1.0:1761315540.683548:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315540.686488:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553856 00000100:00100000:3.0:1761315540.687413:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2212 00000100:00100000:3.0:1761315540.687424:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307553856:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315540.687530:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315540.687884:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307553856:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 460us (4347us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315540.687893:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2212 00010000:00010000:2.0:1761315540.701666:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315541.211693:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315541.211722:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547200:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315541.211819:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315541.211960:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547200 00000100:00100000:2.0:1761315541.212030:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2816 00000100:00100000:2.0:1761315541.212037:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547200:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315541.212083:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315541.215613:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547200:12345-0@lo:41:osp-pre-1-0.0 Request processed in 3563us (3783us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315541.215638:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2816 00000100:00100000:1.0:1761315541.215776:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547200:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315541.724515:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315542.747563:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315543.771526:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315543.771547:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322547264:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315543.771612:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.771668:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547264 00000100:00100000:1.0:1761315543.771718:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 985 00000100:00100000:1.0:1761315543.771726:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055d37d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322547264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315543.771757:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315543.771769:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322547328:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315543.771836:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055d37d5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322547264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 108us (224us total) trans 0 rc 0/0 00000100:00080000:0.0:1761315543.771842:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315543.771847:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 985 00000100:00080000:0.0:1761315543.771849:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315543.771854:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315543.771860:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1761315543.771864:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315543.771870:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315543.771884:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d14b1c65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322547392:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315543.771891:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b52f64ae x1846869322547712/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315543.771910:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315543.771911:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322547456:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315543.771915:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315543.771919:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315543.771922:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3786 00000100:00080000:0.0:1761315543.771925:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315543.771928:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315543.771939:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f159976f x1846869322547776/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315543.771966:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315543.771979:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.772004:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322547840:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315543.772027:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009e962613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547648:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.772039:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315543.772076:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.772088:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547456 00000100:00100000:1.0:1761315543.772133:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547840 00000100:00100000:1.0:1761315543.772167:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4503 00000100:00100000:0.0:1761315543.772171:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322547520:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.772175:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a919a5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322547456:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315543.772219:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.772241:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a919a5d1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322547456:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (270us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315543.772254:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547392 00000100:00100000:0.0:1761315543.772259:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322547904:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315543.772322:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1670 00000100:00100000:3.0:1761315543.772329:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322547392:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315543.772417:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322547392:12345-0@lo:400:kworker/u8:0.0 Request processed in 87us (446us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315543.772424:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1670 00000100:00100000:2.0:1761315543.773029:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4504 00000100:00100000:1.0:1761315543.773040:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4503 00000100:00100000:2.0:1761315543.773041:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5e3a129 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322547840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315543.773110:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5e3a129 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322547840:12345-0@lo:400:kworker/u8:0.0 Request processed in 70us (1071us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315543.773118:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4504 00000100:00100000:1.0:1761315543.773134:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322547840:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773149:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fddf45ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322547456:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315543.773174:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322547712:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315543.773216:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.773220:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547648 00000100:00100000:2.0:1761315543.773244:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f159976f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322547776:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773259:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547520 00000100:00100000:2.0:1761315543.773276:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.773296:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2817 00000100:00100000:1.0:1761315543.773303:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315543.773371:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547712 00000100:00100000:1.0:1761315543.773401:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322547648:12345-0@lo:400:kworker/u8:0.0 Request processed in 93us (1322us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315543.773405:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315543.773406:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547776 00000100:00100000:1.0:1761315543.773418:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2817 00000100:00100000:2.0:1761315543.773438:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2213 00000100:00100000:1.0:1761315543.773446:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2818 00000100:00100000:2.0:1761315543.773447:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322547712:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773456:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322547520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773485:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322547520:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (1267us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315543.773490:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2818 00000100:00100000:1.0:1761315543.773519:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d14b1c65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322547392:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773530:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009e962613 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322547648:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315543.773546:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315543.773564:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315543.773626:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322547520:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773626:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322547264:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315543.773627:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322547712:12345-0@lo:8:kworker/u8:0.0 Request processed in 173us (405us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315543.773655:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2213 00000100:00100000:0.0:1761315543.773662:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315543.773680:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000932fbacb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322547584:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773686:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547328 00000100:00100000:2.0:1761315543.773688:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2214 00000100:00100000:2.0:1761315543.773696:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322547776:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315543.773713:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2819 00000100:00100000:1.0:1761315543.773719:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322547328:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315543.773725:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:1.0:1761315543.773743:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315543.773749:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322547968:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315543.773773:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315543.773786:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315543.773788:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322547328:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (225us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315543.773797:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2819 00000100:00100000:2.0:1761315543.773834:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322547776:12345-0@lo:8:kworker/u8:0.0 Request processed in 132us (553us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315543.773845:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2214 00000100:00100000:1.0:1761315543.773855:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4a6d5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322547328:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315543.773857:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547904 00000100:00100000:2.0:1761315543.773874:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547968 00000100:00100000:2.0:1761315543.773923:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1485 00000100:00100000:2.0:1761315543.773929:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322547968:12345-0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315543.773948:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315543.773981:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322547968:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315543.773990:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1485 00000100:00100000:3.0:1761315543.774153:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4505 00000100:00100000:3.0:1761315543.774160:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322547904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315543.774199:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322547904:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (804us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315543.774205:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4505 00000100:00100000:3.0:1761315543.774222:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322547584 00000100:00100000:2.0:1761315543.778673:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4506 00000100:00100000:2.0:1761315543.778688:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322547584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315543.778791:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322547584:12345-0@lo:400:kworker/u8:0.0 Request processed in 105us (5067us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315543.778799:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4506 00000100:00080000:1.0:1761315543.778839:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315543.778845:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315543.778850:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322547712:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315543.778862:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315543.778865:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315543.778868:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f159976f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322547776:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315543.778914:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000932fbacb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322547584:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.778925:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322547968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315543.778941:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322547904:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315544.795618:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315545.802391:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315545.802601:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315545.803162:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553984 00000100:00100000:1.0:1761315545.803986:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315545.804490:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4507 00000100:00100000:0.0:1761315545.804499:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315545.804598:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4a6d5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307553984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 98us (2213us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315545.804607:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4507 00000100:00100000:2.0:1761315545.804608:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554112 00000100:00100000:2.0:1761315545.804737:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2215 00000100:00100000:2.0:1761315545.804743:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307554112:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:0.0:1761315545.804757:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307553920 00000100:00100000:0.0:1761315545.804788:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1671 00000100:00100000:0.0:1761315545.804792:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315545.804821:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307553920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 28us (838us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315545.804827:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1671 00010000:02020000:2.0:1761315545.804832:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315545.805128:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307554112:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 384us (2527us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315545.805136:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2215 00000100:00100000:1.0:1761315545.805197:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315545.805693:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554048 00000100:00100000:0.0:1761315545.805733:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4508 00000100:00100000:0.0:1761315545.805738:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000932fbacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315545.805771:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000932fbacb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 33us (574us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315545.805778:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4508 00010000:00010000:0.0:1761315545.819869:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315546.333508:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eccc01e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322548032:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315546.333581:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315546.333684:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548032 00000100:00100000:0.0:1761315546.333739:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2820 00000100:00100000:0.0:1761315546.333745:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322548032:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315546.333776:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315546.334711:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322548032:12345-0@lo:41:osp-pre-1-0.0 Request processed in 957us (1122us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315546.334713:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eccc01e8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322548032:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315546.334720:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2820 00010000:00010000:2.0:1761315546.843602:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315547.867999:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315548.891522:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315548.891685:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315548.891690:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315548.891693:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315548.891701:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3791 00000100:00080000:3.0:1761315548.891704:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315548.891709:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315548.891726:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e4e2e24c x1846869322548416/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315548.891739:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315548.891743:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315548.891744:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315548.891747:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3791 00000100:00080000:3.0:1761315548.891749:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315548.891751:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315548.891760:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f926b0a9 x1846869322548480/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315548.891812:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099f66cdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322548160:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.891819:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322548224:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.891882:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.891883:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315548.891922:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322548096:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315548.891964:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be5a8ef4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322548288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.892791:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322548544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.892792:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322548672:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.892824:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315548.892848:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322548864:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315548.892871:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.892907:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315548.892909:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548224 00000100:00100000:2.0:1761315548.892926:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa32dc4d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322548800:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.892956:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315548.892956:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2821 00000100:00100000:3.0:1761315548.892964:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322548224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.892990:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548800 00000100:00100000:2.0:1761315548.893017:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1486 00000100:00100000:2.0:1761315548.893021:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322548800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893460:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322548224:12345-0@lo:400:kworker/u8:0.0 Request processed in 496us (1583us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.893469:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2821 00000100:00100000:1.0:1761315548.893484:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.893507:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322548800:12345-0@lo:400:kworker/u8:0.0 Request processed in 485us (556us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315548.893514:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1486 00000100:00100000:1.0:1761315548.893517:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322548352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893525:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322548224:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.893547:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.893547:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa32dc4d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322548800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.893563:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322548736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.893574:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548160 00000100:00100000:3.0:1761315548.893582:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548352 00000100:00100000:1.0:1761315548.893587:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.893601:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548672 00000100:00100000:3.0:1761315548.893613:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2822 00000100:00100000:1.0:1761315548.893616:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4e2e24c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322548416:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893620:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322548352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.893623:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548544 00000100:00100000:1.0:1761315548.893644:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.893644:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4509 00000100:00100000:2.0:1761315548.893649:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e8c11e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322548160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.893659:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f926b0a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322548480:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893667:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322548352:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (121us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.893674:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2822 00000100:00100000:2.0:1761315548.893695:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e8c11e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322548160:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (1816us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.893697:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4510 00000100:00100000:3.0:1761315548.893702:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6196fc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322548672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315548.893707:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4509 00000100:00100000:2.0:1761315548.893726:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4511 00000100:00100000:2.0:1761315548.893732:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000befc837e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322548544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893737:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6196fc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322548672:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (912us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.893745:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4510 00000100:00100000:2.0:1761315548.893766:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000befc837e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322548544:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (858us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315548.893772:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315548.893773:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4511 00000100:00100000:3.0:1761315548.893785:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322548672:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.893806:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548864 00000100:00100000:2.0:1761315548.893808:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099f66cdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322548160:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893814:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548736 00000100:00100000:2.0:1761315548.893820:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322548544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893839:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548416 00000100:00100000:3.0:1761315548.893855:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548480 00000100:00100000:3.0:1761315548.893863:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1487 00000100:00100000:3.0:1761315548.893867:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000508e2700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322548736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.893896:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000508e2700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322548736:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (308us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315548.893897:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 986 00000100:00100000:2.0:1761315548.893906:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eccc01e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322548864:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315548.893907:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1487 00002000:00000024:2.0:1761315548.893932:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315548.893972:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2216 00000100:00100000:2.0:1761315548.893973:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eccc01e8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322548864:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (1101us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315548.893977:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322548416:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315548.893980:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 986 00000100:00100000:2.0:1761315548.894005:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2217 00000100:00100000:2.0:1761315548.894009:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322548480:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315548.894054:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548096 00000100:00100000:3.0:1761315548.894072:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1672 00000100:00100000:3.0:1761315548.894075:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322548096:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315548.894093:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315548.894099:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322548096:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (616us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.894105:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1672 00000100:00100000:2.0:1761315548.894133:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322548480:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (362us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315548.894141:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2217 00000100:00100000:3.0:1761315548.894156:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322548864:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315548.894261:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315548.894278:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000616e67fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322548608:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315548.894299:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315548.894318:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548608 00010000:02020000:1.0:1761315548.894328:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315548.894337:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2823 00000100:00100000:0.0:1761315548.894355:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322548608:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315548.894377:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315548.894381:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322548416:12345-0@lo:8:kworker/u8:0.0 Request processed in 402us (736us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315548.894389:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2216 00000100:00100000:0.0:1761315548.894408:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322548608:12345-0@lo:41:osp-pre-0-1.0 Request processed in 65us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315548.894415:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2823 00000100:00080000:1.0:1761315548.894418:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315548.894423:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315548.894428:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4e2e24c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322548416:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315548.894439:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315548.894442:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315548.894445:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f926b0a9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322548480:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315548.894452:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000616e67fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322548608:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315548.894483:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322548352:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.894492:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322548736:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315548.894499:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322548096:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.895373:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548288 00000100:00100000:3.0:1761315548.895415:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4512 00000100:00100000:3.0:1761315548.895420:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322548288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315548.895985:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322548288:12345-0@lo:400:kworker/u8:0.0 Request processed in 563us (1722us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315548.895993:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4512 00000100:00100000:0.0:1761315548.896064:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be5a8ef4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322548288:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315549.917515:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315550.923258:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315550.923455:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315550.923497:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315550.923518:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554368 00000100:00100000:2.0:1761315550.923538:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315550.923638:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554240 00000100:00100000:1.0:1761315550.923673:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554304 00000100:00100000:1.0:1761315550.923695:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4513 00000100:00100000:1.0:1761315550.923707:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a966e8fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554240:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315550.923710:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554176 00000100:00100000:3.0:1761315550.923744:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1673 00000100:00100000:1.0:1761315550.923793:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a966e8fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554240:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 85us (543us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315550.923800:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4513 00000100:00100000:1.0:1761315550.923815:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4514 00000100:00100000:1.0:1761315550.923819:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c0c3f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554304:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315550.923841:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c0c3f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554304:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 22us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315550.923847:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4514 00000100:00100000:0.0:1761315550.925240:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2218 00000100:00100000:3.0:1761315550.925341:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315550.927665:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 3906us (4160us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315550.927913:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033e21383 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307554368:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315550.928010:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315550.928076:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033e21383 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307554368:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 167us (4622us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315550.928085:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2218 00000100:00100000:3.0:1761315550.929035:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1673 00010000:00010000:2.0:1761315550.944676:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315551.451627:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315551.451650:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322548928:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315551.451716:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315551.451821:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548928 00000100:00100000:3.0:1761315551.451876:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2824 00000100:00100000:3.0:1761315551.451882:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322548928:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315551.451918:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315551.451975:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322548928:12345-0@lo:41:osp-pre-1-0.0 Request processed in 92us (259us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315551.451982:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2824 00000100:00100000:2.0:1761315551.452109:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322548928:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315551.963634:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315552.987541:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315554.011509:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322548992:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.011580:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315554.011659:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322548992 00000100:00100000:1.0:1761315554.011704:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1674 00000100:00100000:1.0:1761315554.011710:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322548992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315554.013635:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322549056:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315554.013685:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315554.013714:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315554.013720:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322549120:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315554.013744:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315554.013754:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322548992:12345-0@lo:400:kworker/u8:2.0 Request processed in 2043us (2174us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315554.013773:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549056 00000100:00100000:1.0:1761315554.013775:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1674 00000100:00100000:0.0:1761315554.013805:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 987 00000100:00100000:0.0:1761315554.013810:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322549056:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315554.013822:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549184:0@lo:400:kworker/u8:2.0 00002000:00000024:0.0:1761315554.013835:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315554.013845:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315554.013864:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322549056:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (180us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315554.013870:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 987 00000100:00100000:1.0:1761315554.013893:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549120 00000100:00100000:0.0:1761315554.013905:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322549056:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315554.013916:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4515 00000100:00100000:1.0:1761315554.013920:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322549120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315554.013951:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322549120:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (208us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315554.013956:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4515 00010000:00010000:0.0:1761315554.013962:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315554.013974:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549184 00000100:00100000:0.0:1761315554.013976:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322549120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315554.013995:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2825 00000100:00100000:1.0:1761315554.013999:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322549184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315554.014015:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315554.014020:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549248:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315554.014023:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322549184:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315554.014028:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2825 00000100:00100000:0.0:1761315554.014039:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315554.014070:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549184:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315554.014087:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549248 00000100:00100000:0.0:1761315554.014105:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2826 00000100:00100000:0.0:1761315554.014108:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322549248:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315554.014129:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315554.014163:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322549248:12345-0@lo:41:osp-pre-0-1.0 Request processed in 53us (123us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315554.014169:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2826 00000100:00100000:1.0:1761315554.014178:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549248:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315554.014565:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315554.014570:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315554.014573:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315554.014579:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3796 00000100:00080000:3.0:1761315554.014582:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 6s 00000100:00080000:3.0:1761315554.014585:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315554.014605:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008ebd9dcb x1846869322549440/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315554.014618:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315554.014621:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315554.014623:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315554.014625:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3796 00000100:00080000:3.0:1761315554.014627:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s 00000100:00080000:3.0:1761315554.014629:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315554.014640:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000be5a8ef4 x1846869322549504/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315554.014683:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322549568:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315554.014709:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315554.014726:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315554.014730:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322549632:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315554.014746:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315554.014777:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549568 00000100:00100000:3.0:1761315554.014797:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549632 00000100:00100000:3.0:1761315554.014811:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4516 00000100:00100000:3.0:1761315554.014815:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f926b0a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322549568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315554.014843:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f926b0a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322549568:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (134us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315554.014848:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4516 00000100:00100000:3.0:1761315554.014859:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4517 00000100:00100000:3.0:1761315554.014862:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4e2e24c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322549632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315554.014880:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4e2e24c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322549632:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (134us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315554.014885:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4517 00000100:00100000:3.0:1761315554.014911:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322549632:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315554.014919:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322549568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.015029:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322548992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315554.015086:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322549440:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315554.015113:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315554.015939:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000be5a8ef4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322549504:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315554.015967:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315554.015986:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549440 00000100:00100000:2.0:1761315554.016011:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2219 00000100:00100000:2.0:1761315554.016016:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322549440:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315554.016100:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315554.016131:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322549440:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (1018us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315554.016137:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2219 00000100:00100000:2.0:1761315554.016170:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.016191:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315554.016208:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315554.016226:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322549312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.016246:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315554.016259:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549696:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.016282:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549312 00000100:00100000:2.0:1761315554.016305:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4518 00000100:00100000:2.0:1761315554.016308:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322549312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.016359:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322549312:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (95us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315554.016366:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4518 00000100:00100000:2.0:1761315554.016386:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549376 00000100:00100000:2.0:1761315554.016404:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2827 00000100:00100000:2.0:1761315554.016407:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093968913 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.016432:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093968913 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549376:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (241us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315554.016437:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2827 00000100:00080000:1.0:1761315554.016583:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315554.016587:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315554.016591:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322549440:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315554.016655:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549504 00000100:00100000:3.0:1761315554.016685:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2220 00000100:00100000:3.0:1761315554.016689:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322549504:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315554.019785:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315554.019889:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322549504:12345-0@lo:8:kworker/u8:2.0 Request processed in 3196us (3919us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315554.019902:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2220 00000100:00080000:3.0:1761315554.019969:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315554.019973:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315554.019978:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000be5a8ef4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322549504:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315554.020264:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315554.020310:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c60bc2b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315554.020330:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d38b7cc5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322549312:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315554.020413:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549696 00000100:00100000:1.0:1761315554.020463:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1488 00000100:00100000:1.0:1761315554.020468:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315554.020527:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549696:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315554.020533:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1488 00000100:00100000:2.0:1761315554.020599:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549696:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315555.035492:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315556.040837:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315556.040890:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315556.041004:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554496 00000100:00100000:0.0:1761315556.041064:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4519 00000100:00100000:0.0:1761315556.041070:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315556.041155:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 84us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315556.041164:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4519 00000100:00100000:3.0:1761315556.041292:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315556.041464:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554560 00000100:00100000:2.0:1761315556.041490:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4520 00000100:00100000:2.0:1761315556.041494:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000be5a8ef4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307554560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315556.041523:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000be5a8ef4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307554560:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 28us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315556.041530:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4520 00000100:00100000:1.0:1761315556.041748:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554432 00000100:00100000:1.0:1761315556.041821:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1675 00000100:00100000:1.0:1761315556.041825:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554432:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315556.041852:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554432:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (1023us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315556.041859:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1675 00000100:00100000:3.0:1761315556.041958:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315556.041982:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554624 00000100:00100000:3.0:1761315556.042003:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2221 00000100:00100000:3.0:1761315556.042007:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307554624:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315556.042099:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315556.042125:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307554624:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 116us (168us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315556.042131:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2221 00010000:00010000:2.0:1761315556.061732:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315556.571596:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549760:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315556.571658:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315556.571810:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549760 00000100:00100000:0.0:1761315556.572706:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2828 00000100:00100000:0.0:1761315556.572714:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549760:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315556.572747:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315556.573232:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322549760:12345-0@lo:41:osp-pre-1-0.0 Request processed in 516us (1573us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315556.573240:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2828 00000100:00100000:2.0:1761315556.573340:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322549760:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315557.083618:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315558.107757:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315559.131589:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315559.131658:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315559.131663:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315559.131667:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315559.131674:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3801 00000100:00080000:0.0:1761315559.131678:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 11s 00000100:00080000:0.0:1761315559.131683:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315559.131702:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000077de439a x1846869322550272/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315559.131718:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315559.131721:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315559.131722:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315559.131725:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3801 00000100:00080000:0.0:1761315559.131727:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s 00000100:00080000:0.0:1761315559.131731:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315559.131742:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000057deb555 x1846869322550336/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315559.131748:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322549952:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.131791:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322550016:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315559.131872:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549824:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315559.131934:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322549888:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315559.132397:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132422:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000522c807b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322550400:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132442:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132460:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315559.132464:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fce761a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322550592:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132484:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132506:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322550272:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132526:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132541:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322550336:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132551:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132562:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315559.132579:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322550208:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132601:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550016 00000100:00100000:1.0:1761315559.132610:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315559.132644:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550400 00000100:00100000:1.0:1761315559.132653:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549952 00000100:00100000:0.0:1761315559.132669:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4521 00000100:00100000:0.0:1761315559.132675:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d76b5693 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322550016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132686:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676 00000100:00100000:1.0:1761315559.132692:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322549952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132730:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d76b5693 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322550016:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (334us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315559.132735:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322549952:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315559.132739:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4521 00000100:00100000:1.0:1761315559.132746:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676 00000100:00100000:0.0:1761315559.132760:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4522 00000100:00100000:0.0:1761315559.132765:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7ebf84e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322550400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132791:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bc9be46d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322549952:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.132798:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7ebf84e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322550400:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (354us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315559.132809:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4522 00000100:00100000:1.0:1761315559.132839:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550592 00000100:00100000:0.0:1761315559.132862:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d733c02e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322550016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132866:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550272 00000100:00100000:0.0:1761315559.132874:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000522c807b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322550400:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132883:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550336 00000100:00100000:1.0:1761315559.132900:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1489 00000100:00100000:1.0:1761315559.132905:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ded54f3d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322550592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.132938:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ded54f3d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322550592:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (454us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315559.132944:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1489 00000100:00100000:1.0:1761315559.132955:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2222 00000100:00100000:1.0:1761315559.132958:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322550272:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315559.133023:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315559.133041:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315559.133042:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000727dca98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322550080:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315559.133078:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315559.133086:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322550272:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (558us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315559.133094:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2222 00000100:00100000:2.0:1761315559.133099:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322550464:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.133107:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2223 00000100:00100000:1.0:1761315559.133111:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322550336:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315559.133123:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315559.133165:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549824 00010000:02020000:1.0:1761315559.133202:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315559.133206:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 988 00000100:00100000:2.0:1761315559.133211:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322549824:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315559.133233:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315559.133237:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322550336:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (675us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315559.133245:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2223 00000100:00100000:1.0:1761315559.133275:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550464 00000100:00100000:2.0:1761315559.133276:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322549824:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (252us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315559.133283:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 988 00000100:00100000:2.0:1761315559.133303:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550080 00000100:00100000:2.0:1761315559.133325:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2829 00000100:00100000:2.0:1761315559.133328:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322550080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315559.133372:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322550080:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315559.133377:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2829 00000100:00100000:2.0:1761315559.133412:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322549824:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315559.133424:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000727dca98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322550080:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.133460:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4523 00000100:00100000:1.0:1761315559.133464:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322550464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315559.133499:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322550464:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (377us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315559.133536:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315559.133562:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322550144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315559.133588:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315559.133600:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322550528:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315559.133620:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315559.133646:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550528 00000100:00100000:3.0:1761315559.133666:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1490 00000100:00100000:3.0:1761315559.133670:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322550528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315559.133708:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322550528:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (87us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315559.133713:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1490 00000100:00100000:3.0:1761315559.133739:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322549888 00000100:00100000:3.0:1761315559.133756:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2830 00000100:00100000:3.0:1761315559.133759:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005522250c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322549888:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315559.133776:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315559.133799:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005522250c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322549888:12345-0@lo:41:osp-pre-0-1.0 Request processed in 40us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315559.133804:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2830 00000100:00100000:3.0:1761315559.133820:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550144 00000100:00100000:1.0:1761315559.134242:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4523 00000100:00080000:1.0:1761315559.134291:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315559.134294:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315559.134297:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322550272:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315559.134306:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315559.134309:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315559.134311:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322550336:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315559.134337:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fce761a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322550592:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315559.134453:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322550464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315559.134493:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322549888:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315559.134502:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322550528:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.134660:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550208 00000100:00100000:0.0:1761315559.134688:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2831 00000100:00100000:0.0:1761315559.134692:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322550208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315559.134733:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322550208:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (2123us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315559.134739:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2831 00000100:00100000:0.0:1761315559.134765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322550208:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315559.134796:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4524 00000100:00100000:3.0:1761315559.134801:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099f66cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322550144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315559.134943:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099f66cdd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322550144:12345-0@lo:400:kworker/u8:0.0 Request processed in 140us (1354us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315559.134951:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4524 00000100:00100000:0.0:1761315559.135096:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322550144:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315560.158225:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315561.162803:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315561.163649:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554752 00000100:00100000:2.0:1761315561.163688:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315561.163713:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4525 00000100:00100000:3.0:1761315561.163720:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315561.163721:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315561.163744:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315561.163770:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554880 00000100:00100000:3.0:1761315561.163790:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307554752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 70us (995us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315561.163791:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2224 00000100:00100000:2.0:1761315561.163795:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307554880:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:3.0:1761315561.163797:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4525 00000100:00100000:3.0:1761315561.163812:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554816 00000100:00100000:1.0:1761315561.163832:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554688 00000100:00100000:3.0:1761315561.163833:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4526 00000100:00100000:3.0:1761315561.163839:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315561.163869:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307554816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 27us (145us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315561.163870:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1677 00000100:00100000:1.0:1761315561.163882:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315561.163883:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4526 00010000:02020000:2.0:1761315561.163886:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315561.163917:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307554880:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 120us (172us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315561.163927:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307554688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 46us (237us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315561.163927:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2224 00000100:00100000:1.0:1761315561.163938:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1677 00010000:00010000:0.0:1761315561.179536:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315561.691689:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322550656:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315561.691757:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315561.692600:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550656 00000100:00100000:2.0:1761315561.692676:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2832 00000100:00100000:2.0:1761315561.692683:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322550656:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315561.692716:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315561.692770:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322550656:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (1014us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315561.692776:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2832 00000100:00100000:1.0:1761315561.692854:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322550656:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315562.205557:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315563.227859:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315564.251552:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315564.251559:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315564.251563:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315564.251570:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3806 00000100:00080000:0.0:1761315564.251574:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 16s 00000100:00080000:0.0:1761315564.251580:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315564.251598:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a476cb8a x1846869322551040/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315564.251609:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315564.251612:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315564.251613:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315564.251615:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3806 00000100:00080000:0.0:1761315564.251617:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s 00000100:00080000:0.0:1761315564.251619:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315564.251625:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ec256aee x1846869322551104/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315564.251665:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b898b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322550848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.251720:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.251740:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dab8c06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322551232:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.251758:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.251775:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315564.251779:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004580caf3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322550912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.251793:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.251802:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322551296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.251822:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.251851:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550848 00000100:00100000:0.0:1761315564.251893:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2833 00000100:00100000:0.0:1761315564.251899:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322550848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.251943:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322550848:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (223us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.251950:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2833 00000100:00100000:0.0:1761315564.251972:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551232 00000100:00100000:0.0:1761315564.251996:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550912 00000100:00100000:0.0:1761315564.252013:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4527 00000100:00100000:0.0:1761315564.252016:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322551232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252044:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000408c4763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322551232:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252050:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4527 00000100:00100000:0.0:1761315564.252060:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4528 00000100:00100000:0.0:1761315564.252064:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322550912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252088:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000aa10021 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322550912:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (295us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252093:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4528 00000100:00100000:0.0:1761315564.252133:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004580caf3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322550912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252155:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b898b67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322550848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252162:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dab8c06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322551232:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315564.252220:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315564.252227:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322550784:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252248:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.252258:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f69efca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322551168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252272:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.252282:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1761315564.252287:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322550720:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252306:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.252316:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071a6088f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322550976:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252334:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.252366:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322551360:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252383:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315564.252405:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550784 00000100:00100000:0.0:1761315564.252422:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551168 00000100:00100000:0.0:1761315564.252434:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4529 00000100:00100000:0.0:1761315564.252437:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dab8c06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322550784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252461:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dab8c06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322550784:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (213us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252466:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4529 00000100:00100000:0.0:1761315564.252474:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4530 00000100:00100000:0.0:1761315564.252477:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b898b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322551168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252494:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b898b67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322551168:12345-0@lo:400:kworker/u8:0.0 Request processed in 16us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252498:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4530 00000100:00100000:0.0:1761315564.252509:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550976 00000100:00100000:0.0:1761315564.252525:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2834 00000100:00100000:0.0:1761315564.252528:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322550976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252547:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322550976:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (213us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252552:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2834 00000100:00100000:0.0:1761315564.252572:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071a6088f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322550976:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252579:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322550784:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252586:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f69efca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322551168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252622:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551296 00000100:00100000:0.0:1761315564.252640:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551360 00000100:00100000:0.0:1761315564.252651:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1491 00000100:00100000:0.0:1761315564.252654:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322551296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252679:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322551296:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (857us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252685:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1491 00000100:00100000:0.0:1761315564.252693:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1492 00000100:00100000:0.0:1761315564.252696:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322551360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252714:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322551360:12345-0@lo:400:kworker/u8:0.0 Request processed in 18us (331us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315564.252719:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1492 00000100:00100000:0.0:1761315564.252737:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e2133c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322551296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315564.252753:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061a066c1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322551360:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315564.252887:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a476cb8a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322551040:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315564.252916:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315564.252932:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec256aee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322551104:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315564.252949:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315564.252971:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322550720 00000100:00100000:1.0:1761315564.252994:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1678 00000100:00100000:1.0:1761315564.252997:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322550720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315564.253030:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322550720:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (724us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315564.253036:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1678 00000100:00100000:1.0:1761315564.253073:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551040 00000100:00100000:1.0:1761315564.253092:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551104 00000100:00100000:1.0:1761315564.253113:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2225 00000100:00100000:1.0:1761315564.253117:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322551040:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315564.253168:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2226 00000100:00100000:3.0:1761315564.253173:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322551104:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315564.253283:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315564.253318:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322551040:12345-0@lo:8:kworker/u8:0.0 Request processed in 200us (401us total) trans 0 rc -19/-19 00010000:02020000:3.0:1761315564.253323:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315564.253327:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2225 00000100:00080000:1.0:1761315564.253383:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315564.253387:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315564.253393:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a476cb8a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322551040:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315564.253400:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322551104:12345-0@lo:8:kworker/u8:0.0 Request processed in 223us (448us total) trans 0 rc -19/-19 00000100:00080000:1.0:1761315564.253404:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315564.253408:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315564.253411:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2226 00000100:00100000:1.0:1761315564.253412:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec256aee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322551104:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315564.253458:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322550720:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315564.253720:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315564.253727:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322551424:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315564.253750:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315564.253773:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551424 00000100:00100000:1.0:1761315564.253793:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2835 00000100:00100000:1.0:1761315564.253796:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322551424:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315564.253819:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315564.253844:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322551424:12345-0@lo:41:osp-pre-0-1.0 Request processed in 47us (94us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315564.253849:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2835 00000100:00100000:1.0:1761315564.253869:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322551424:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315564.254011:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322551488:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315564.254037:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315564.254095:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551488 00000100:00100000:2.0:1761315564.254122:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 989 00000100:00100000:2.0:1761315564.254126:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322551488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315564.254157:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315564.255246:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322551488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1116us (1207us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315564.255257:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 989 00000100:00100000:1.0:1761315564.255326:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000085790e43 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322551488:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315565.275605:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315566.280181:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315566.280229:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315566.280266:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315566.280444:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307554944 00000100:00100000:1.0:1761315566.280491:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1679 00000100:00100000:1.0:1761315566.280497:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307554944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315566.280564:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307554944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 66us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315566.280572:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1679 00000100:00100000:2.0:1761315566.282666:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555008 00000100:00100000:2.0:1761315566.282724:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555072 00000100:00100000:2.0:1761315566.282756:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4531 00000100:00100000:2.0:1761315566.282762:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d38b7cc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307555008:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315566.282829:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d38b7cc5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307555008:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (2656us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315566.282836:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4531 00000100:00100000:2.0:1761315566.282853:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4532 00000100:00100000:2.0:1761315566.282858:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c60bc2b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307555072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315566.282883:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c60bc2b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307555072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (2618us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315566.282889:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4532 00000100:00100000:1.0:1761315566.285434:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315566.285526:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555136 00000100:00100000:2.0:1761315566.285608:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2227 00000100:00100000:2.0:1761315566.285615:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307555136:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315566.285708:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315566.285771:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307555136:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 156us (344us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315566.285779:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2227 00010000:00010000:0.0:1761315566.299560:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315566.813041:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315566.813141:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322551552:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315566.813207:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315566.813307:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551552 00000100:00100000:1.0:1761315566.813376:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2836 00000100:00100000:1.0:1761315566.813382:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322551552:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315566.813419:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315566.814031:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322551552:12345-0@lo:41:osp-pre-1-0.0 Request processed in 647us (823us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315566.814040:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2836 00000100:00100000:2.0:1761315566.814113:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ebd9dcb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322551552:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315567.323665:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315568.347598:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315569.371576:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322551680:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315569.371649:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315569.372332:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551680 00000100:00100000:2.0:1761315569.372332:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315569.372341:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322551616:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315569.372398:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315569.372402:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2837 00000100:00100000:1.0:1761315569.372409:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322551680:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315569.372430:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551616 00000004:00000024:1.0:1761315569.372439:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315569.372460:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 990 00000100:00100000:2.0:1761315569.372465:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322551616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315569.372485:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315569.372489:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322551680:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (840us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315569.372497:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2837 00000100:00100000:2.0:1761315569.372517:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322551616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (118us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.372524:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 990 00000100:00100000:2.0:1761315569.372561:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf961e73 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322551616:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315569.372571:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322551680:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315569.372593:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322551744:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315569.372621:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315569.372622:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315569.372628:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315569.372631:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315569.372638:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3811 00000100:00080000:1.0:1761315569.372642:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761315569.372642:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1761315569.372648:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315569.372650:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322551808:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.372671:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a790d12c x1846869322552000/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315569.372684:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315569.372690:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315569.372694:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315569.372696:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315569.372700:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3811 00000100:00080000:1.0:1761315569.372703:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315569.372706:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315569.372716:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551744 00000100:00100000:1.0:1761315569.372725:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000885db3f x1846869322552064/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315569.372745:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680 00000100:00100000:2.0:1761315569.372751:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322551744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.372782:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322552192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.372786:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322551744:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (163us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.372794:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680 00000100:00100000:1.0:1761315569.372816:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315569.372836:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551808 00000100:00100000:1.0:1761315569.372841:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315569.372847:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322552128:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.372862:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552192 00000100:00100000:1.0:1761315569.372872:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315569.372889:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552128 00000100:00100000:1.0:1761315569.372911:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322552000:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315569.372912:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4533 00000100:00100000:2.0:1761315569.372918:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000616e67fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322551808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.372945:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315569.372955:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000616e67fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322551808:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (275us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.372961:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4533 00000100:00100000:2.0:1761315569.372970:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4534 00000100:00100000:2.0:1761315569.372974:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322552192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315569.373014:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322551872:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373474:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322552064:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315569.373490:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322552192:12345-0@lo:400:kworker/u8:0.0 Request processed in 515us (676us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.373497:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4534 00000100:00100000:1.0:1761315569.373510:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315569.373513:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4535 00000100:00100000:2.0:1761315569.373517:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322552128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.373545:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322552128:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (675us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.373550:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4535 00000100:00100000:1.0:1761315569.373565:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552000 00000100:00100000:2.0:1761315569.373572:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322551936:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373590:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552064 00000100:00100000:2.0:1761315569.373596:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315569.373616:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2228 00000100:00100000:1.0:1761315569.373621:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322552000:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315569.373630:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551936 00000100:00100000:2.0:1761315569.373652:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2838 00000100:00100000:2.0:1761315569.373655:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322551936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.373682:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322551936:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (87us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.373687:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2838 00010000:02020000:1.0:1761315569.373707:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315569.373714:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322551936:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373747:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322552000:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (802us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315569.373755:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2228 00000100:00100000:2.0:1761315569.373763:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2229 00000100:00100000:2.0:1761315569.373769:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322552064:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373788:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322552128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373797:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322552192:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315569.373818:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315569.373821:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315569.373823:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322552000:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315569.373843:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322551808:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315569.373849:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315569.373852:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322551744:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.373890:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322552064:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (382us total) trans 0 rc -19/-19 00000100:00080000:1.0:1761315569.373897:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315569.373897:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2229 00000100:00080000:1.0:1761315569.373902:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315569.373907:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322552064:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315569.373983:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315569.374014:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000727dca98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322552256:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315569.374033:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315569.374075:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552256 00010000:00010000:0.0:1761315569.374115:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315569.374115:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1493 00000100:00100000:3.0:1761315569.374120:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322552256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315569.374170:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322552256:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (136us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315569.374177:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1493 00000100:00100000:2.0:1761315569.374183:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322551872 00000100:00100000:2.0:1761315569.374210:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4536 00000100:00100000:2.0:1761315569.374215:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322551872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315569.374243:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e2133c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322551872:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (262us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315569.374248:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4536 00000100:00100000:0.0:1761315569.374303:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322551872:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315569.374311:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000727dca98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322552256:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315570.395767:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315571.406430:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315571.406485:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315571.406539:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315571.406582:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555264 00000100:00100000:3.0:1761315571.406628:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555328 00000100:00100000:3.0:1761315571.406649:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4537 00000100:00100000:3.0:1761315571.406655:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf961e73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307555264:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315571.406723:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4538 00000100:00100000:2.0:1761315571.406731:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ebd9dcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307555328:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315571.406736:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf961e73 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307555264:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (311us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315571.406745:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4537 00000100:00100000:2.0:1761315571.406773:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ebd9dcb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307555328:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 41us (234us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315571.406781:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4538 00000100:00100000:1.0:1761315571.407120:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555200 00000100:00100000:1.0:1761315571.407160:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1681 00000100:00100000:1.0:1761315571.407164:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307555200:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315571.407194:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307555200:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 30us (710us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315571.407200:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1681 00000100:00100000:3.0:1761315571.408572:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315571.408637:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555392 00000100:00100000:2.0:1761315571.408676:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2230 00000100:00100000:2.0:1761315571.408683:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307555392:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315571.408776:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315571.408833:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307555392:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 149us (265us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315571.408843:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2230 00010000:00010000:0.0:1761315571.419525:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315571.933479:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322552320:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315571.933544:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315571.933651:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552320 00000100:00100000:3.0:1761315571.933702:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2839 00000100:00100000:3.0:1761315571.933708:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322552320:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315571.933740:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315571.933795:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322552320:12345-0@lo:41:osp-pre-1-0.0 Request processed in 86us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315571.933803:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2839 00000100:00100000:0.0:1761315571.934396:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322552320:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315572.444204:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315573.468132:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315574.491512:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315574.491631:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315574.491635:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315574.491639:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761315574.491639:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071a6088f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322552512:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315574.491649:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3816 00000100:00080000:0.0:1761315574.491652:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315574.491656:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315574.491679:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005ee7850a x1846869322552768/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315574.491694:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315574.491697:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315574.491699:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315574.491702:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3816 00000100:00080000:0.0:1761315574.491704:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315574.491707:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315574.491711:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315574.491720:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008fce761a x1846869322552832/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315574.491748:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000042bf6d90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322552896:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.491774:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315574.491780:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322552576:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.491812:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552512 00000100:00100000:0.0:1761315574.491819:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315574.491843:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5a9c309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322552960:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.491850:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552896 00000100:00100000:0.0:1761315574.491870:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315574.491874:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4539 00000100:00100000:2.0:1761315574.491880:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322552512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315574.491916:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322552384:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.491928:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322552512:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (218us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315574.491935:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4539 00000100:00100000:2.0:1761315574.491950:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552960 00000100:00100000:0.0:1761315574.491954:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315574.491974:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4540 00000100:00100000:0.0:1761315574.491977:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a476cb8a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322552704:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.491981:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322552896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315574.492019:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315574.492020:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322552896:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (244us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315574.492027:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4540 00000100:00100000:2.0:1761315574.492042:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4541 00000100:00100000:0.0:1761315574.492044:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322553088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.492048:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a9a33f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322552960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315574.492094:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315574.492100:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a9a33f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322552960:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315574.492106:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4541 00000100:00100000:2.0:1761315574.492143:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553088 00000100:00100000:2.0:1761315574.492167:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1494 00000100:00100000:2.0:1761315574.492170:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322553088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315574.492269:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552576 00000100:00100000:3.0:1761315574.492294:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552704 00000100:00100000:3.0:1761315574.492309:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2840 00000100:00100000:3.0:1761315574.492313:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322552576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315574.492363:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322552576:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (524us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315574.492369:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2840 00000100:00100000:3.0:1761315574.492380:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2841 00000100:00100000:3.0:1761315574.492383:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322552704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492466:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322552768:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492495:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.492513:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008fce761a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322552832:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492532:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.492561:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552768 00000100:00100000:1.0:1761315574.492579:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552832 00000100:00100000:1.0:1761315574.492590:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2231 00000100:00100000:1.0:1761315574.492593:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322552768:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315574.492679:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315574.492709:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322552768:12345-0@lo:8:kworker/u8:2.0 Request processed in 114us (213us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315574.492715:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2231 00000100:00100000:1.0:1761315574.492726:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2232 00000100:00100000:1.0:1761315574.492728:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000977da04 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322552832:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315574.492797:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315574.492814:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000977da04 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322552832:12345-0@lo:8:kworker/u8:2.0 Request processed in 85us (282us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315574.492818:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2232 00000100:00080000:1.0:1761315574.492853:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315574.492856:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315574.492859:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322552768:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315574.492865:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315574.492867:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315574.492868:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008fce761a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322552832:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492895:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322552448:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315574.492913:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.492921:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec256aee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322552640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492933:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.492943:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f9a6d13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.492957:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.492974:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552640 00000100:00100000:1.0:1761315574.492992:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4542 00000100:00100000:1.0:1761315574.492995:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b35f622a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322552640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493018:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b35f622a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322552640:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (85us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315574.493023:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4542 00000100:00100000:1.0:1761315574.493036:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553024 00000100:00100000:1.0:1761315574.493048:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1495 00000100:00100000:1.0:1761315574.493051:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2fff388 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322553024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493084:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2fff388 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322553024:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (127us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315574.493089:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1495 00000100:00100000:1.0:1761315574.493109:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec256aee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322552640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493116:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f9a6d13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493150:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552384 00000100:00100000:1.0:1761315574.493168:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1682 00000100:00100000:1.0:1761315574.493171:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322552384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493193:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322552384:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1243us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315574.493198:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1682 00000100:00100000:1.0:1761315574.493225:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322553152:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315574.493241:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315574.493260:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000042bf6d90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322552896:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493268:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071a6088f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322552512:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493291:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322552448 00000100:00100000:1.0:1761315574.493302:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2842 00000100:00100000:1.0:1761315574.493305:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322552448:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315574.493326:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315574.493374:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322552448:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (460us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315574.493382:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2842 00000100:00100000:1.0:1761315574.493402:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072fdd2e7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322552448:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315574.493419:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322552576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493428:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5a9c309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322552960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315574.493452:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553152 00000100:00100000:1.0:1761315574.493469:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 991 00000100:00100000:1.0:1761315574.493472:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322553152:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315574.493488:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315574.493511:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322553152:12345-0@lo:13:osp-pre-1-1.0 Request processed in 39us (270us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315574.493516:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 991 00000100:00100000:1.0:1761315574.493541:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322553152:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315574.495416:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322553088:12345-0@lo:400:kworker/u8:2.0 Request processed in 3239us (3319us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315574.495427:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322552384:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315574.495432:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1494 00000100:00100000:3.0:1761315574.495529:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322552704:12345-0@lo:400:kworker/u8:2.0 Request processed in 3144us (3513us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315574.495537:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2841 00000100:00100000:0.0:1761315574.495747:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a476cb8a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322552704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315574.495762:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322553088:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315575.515648:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315576.520042:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315576.520122:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315576.520153:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315576.520183:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555456 00000100:00100000:2.0:1761315576.520228:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555520 00000100:00100000:3.0:1761315576.520241:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1683 00000100:00100000:3.0:1761315576.520249:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307555456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315576.520261:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555584 00000100:00100000:2.0:1761315576.520278:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4543 00000100:00100000:2.0:1761315576.520282:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307555520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315576.520325:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307555456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (199us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315576.520326:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072fdd2e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307555520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 37us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315576.520370:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4543 00000100:00100000:3.0:1761315576.520374:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1683 00000100:00100000:2.0:1761315576.520403:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4544 00000100:00100000:2.0:1761315576.520409:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7de1df4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307555584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315576.520433:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7de1df4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307555584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (281us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315576.520440:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4544 00000100:00100000:3.0:1761315576.520591:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315576.520677:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555648 00000100:00100000:0.0:1761315576.520701:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2233 00000100:00100000:0.0:1761315576.520715:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307555648:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315576.520813:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315576.520843:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307555648:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 127us (254us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315576.520850:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2233 00010000:00010000:2.0:1761315576.543121:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315577.051609:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322553216:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315577.051674:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315577.051747:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553216 00000100:00100000:0.0:1761315577.051800:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2843 00000100:00100000:0.0:1761315577.051805:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322553216:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315577.051836:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315577.053934:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322553216:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2121us (2253us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315577.054150:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322553216:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315577.054250:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2843 00010000:00010000:0.0:1761315577.563567:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315578.588216:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315579.611802:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd8888b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553280:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315579.611871:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.611912:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315579.611918:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315579.611948:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315579.612438:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315579.612443:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315579.612446:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315579.612451:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3822 00000100:00080000:2.0:1761315579.612453:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315579.612458:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315579.612480:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d48b7a5 x1846869322553664/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315579.612495:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315579.612498:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315579.612499:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315579.612502:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3822 00000100:00080000:2.0:1761315579.612504:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315579.612506:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315579.612517:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000088984be5 x1846869322553728/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315579.612553:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322553408:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612586:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.612601:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b1b735c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322553792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612622:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.612640:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315579.612645:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322553472:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612664:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.612677:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322553856:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612696:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.612726:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553792 00000100:00100000:2.0:1761315579.612765:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553472 00000100:00100000:2.0:1761315579.612780:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553856 00000100:00100000:2.0:1761315579.612798:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4545 00000100:00100000:2.0:1761315579.612804:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322553792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612851:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007be8cc62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322553792:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (228us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.612858:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4545 00000100:00100000:2.0:1761315579.612873:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4546 00000100:00100000:2.0:1761315579.612876:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322553472:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612901:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008f2202f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322553472:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (237us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.612906:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4546 00000100:00100000:2.0:1761315579.612917:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4547 00000100:00100000:2.0:1761315579.612920:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a748b8e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322553856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.612944:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a748b8e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322553856:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (247us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.612949:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4547 00000100:00100000:2.0:1761315579.612994:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322553472:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613004:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322553856:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613011:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b1b735c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322553792:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613063:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322553536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613084:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.613099:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322553920:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613120:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.613138:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315579.613143:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322553600:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613163:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.613188:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553280 00000100:00100000:2.0:1761315579.613213:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553600 00000100:00100000:2.0:1761315579.613229:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2844 00000100:00100000:2.0:1761315579.613233:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322553280:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315579.613257:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315579.613291:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322553280:12345-0@lo:41:osp-pre-0-1.0 Request processed in 57us (1421us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.613297:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2844 00000100:00100000:2.0:1761315579.613307:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2845 00000100:00100000:2.0:1761315579.613311:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322553600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613337:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322553600:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (174us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.613364:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2845 00000100:00100000:2.0:1761315579.613382:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553536 00000100:00100000:2.0:1761315579.613402:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4548 00000100:00100000:2.0:1761315579.613405:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1b735c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322553536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613435:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1b735c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322553536:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.613441:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4548 00000100:00100000:2.0:1761315579.613464:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322553600:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613474:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322553536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613486:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd8888b0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553280:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315579.613531:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553920 00000100:00100000:2.0:1761315579.613556:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1496 00000100:00100000:2.0:1761315579.613559:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322553920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613590:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322553920:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (470us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.613596:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1496 00000100:00100000:2.0:1761315579.613614:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553408 00000100:00100000:2.0:1761315579.613632:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684 00000100:00100000:2.0:1761315579.613636:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322553408:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613667:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322553408:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (1082us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315579.613672:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684 00000100:00100000:2.0:1761315579.613696:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322553664:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613717:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.613730:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322553728:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315579.613750:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315579.613778:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553664 00000100:00100000:2.0:1761315579.613794:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553728 00000100:00100000:2.0:1761315579.613805:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2234 00000100:00100000:2.0:1761315579.613807:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322553664:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315579.613893:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315579.613921:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322553664:12345-0@lo:8:kworker/u8:0.0 Request processed in 112us (204us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315579.613930:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2234 00000100:00100000:2.0:1761315579.613939:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2235 00000100:00100000:2.0:1761315579.613941:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322553728:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315579.614013:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315579.614048:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322553728:12345-0@lo:8:kworker/u8:0.0 Request processed in 105us (297us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315579.614053:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2235 00000100:00080000:2.0:1761315579.614076:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315579.614080:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315579.614083:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322553664:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315579.614090:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315579.614092:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315579.614094:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322553728:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315579.614125:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322553408:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315579.614146:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322553920:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315579.615257:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553344 00000100:00100000:1.0:1761315579.615314:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 992 00000100:00100000:1.0:1761315579.615319:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322553344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315579.615370:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315579.615446:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322553344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 126us (3498us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315579.615454:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 992 00000100:00100000:0.0:1761315579.615595:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322553344:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1761315579.620471:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315580.635498:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315581.641158:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315581.641199:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315581.641228:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315581.641831:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555712 00000100:00100000:3.0:1761315581.641878:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1685 00000100:00100000:3.0:1761315581.641885:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307555712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315581.642048:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555840 00000100:00100000:2.0:1761315581.642083:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4549 00000100:00100000:2.0:1761315581.642088:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001a76876 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307555840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315581.642219:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307555712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 333us (1068us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315581.642227:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1685 00000100:00100000:0.0:1761315581.642255:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555776 00000100:00100000:0.0:1761315581.642282:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4550 00000100:00100000:0.0:1761315581.642285:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a2f4962 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307555776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315581.643068:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315581.643093:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555904 00000100:00100000:1.0:1761315581.643115:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2236 00000100:00100000:1.0:1761315581.643119:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307555904:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315581.643205:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315581.643242:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307555904:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 122us (176us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315581.643248:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2236 00000100:00100000:0.0:1761315581.643299:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a2f4962 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307555776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1010us (2069us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315581.643307:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4550 00000100:00100000:2.0:1761315581.643521:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001a76876 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307555840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1431us (2321us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315581.643530:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4549 00010000:00010000:0.0:1761315581.659488:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315582.171627:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322553984:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315582.171701:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315582.171786:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322553984 00000100:00100000:1.0:1761315582.171848:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2846 00000100:00100000:1.0:1761315582.171855:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322553984:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315582.171887:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315582.172764:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322553984:12345-0@lo:41:osp-pre-1-0.0 Request processed in 907us (1062us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315582.172773:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2846 00000100:00100000:2.0:1761315582.172850:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322553984:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315582.685290:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315583.707532:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315584.731543:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315584.731704:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315584.731724:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322554048:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315584.731789:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315584.731876:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554048 00000100:00100000:3.0:1761315584.731929:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 993 00000100:00100000:3.0:1761315584.731935:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322554048:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315584.731966:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315584.732018:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322554048:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (228us total) trans 0 rc 0/0 00000100:00080000:2.0:1761315584.732019:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315584.732029:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1761315584.732031:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 993 00000100:00080000:2.0:1761315584.732033:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315584.732040:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3827 00000100:00080000:2.0:1761315584.732043:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315584.732047:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315584.732066:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005dd56348 x1846869322554432/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315584.732076:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c8817063 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322554304:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315584.732083:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315584.732087:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315584.732089:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315584.732107:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315584.732175:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322554432:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315584.732205:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322554240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732213:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315584.732241:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315584.732252:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322554176:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.732276:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554432 00000100:00100000:0.0:1761315584.732282:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315584.732302:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2237 00000100:00100000:0.0:1761315584.732304:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322554496:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315584.732309:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2bd8cd7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322554432:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732333:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315584.732387:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554240 00000100:00100000:0.0:1761315584.732410:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554496 00010000:02020000:1.0:1761315584.732417:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315584.732434:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2847 00000100:00100000:0.0:1761315584.732439:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322554240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.732452:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2bd8cd7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322554432:12345-0@lo:8:kworker/u8:0.0 Request processed in 143us (239us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315584.732459:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2237 00000100:00100000:0.0:1761315584.732484:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322554240:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315584.732487:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2848 00000100:00100000:1.0:1761315584.732494:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b50733c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322554496:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315584.732497:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2847 00000004:00000024:1.0:1761315584.732520:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315584.732523:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554304 00000100:00100000:0.0:1761315584.732549:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554176 00000100:00100000:1.0:1761315584.732554:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b50733c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322554496:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (221us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315584.732560:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2848 00000100:00100000:0.0:1761315584.732571:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4551 00000100:00100000:0.0:1761315584.732575:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322554304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732610:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028fcbdf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322554304:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (503us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315584.732617:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4551 00000100:00100000:0.0:1761315584.732636:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4552 00000100:00100000:0.0:1761315584.732640:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e0281a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322554176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732671:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e0281a9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322554176:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (390us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315584.732677:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4552 00000100:00100000:0.0:1761315584.732731:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322554240:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315584.732762:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315584.732766:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315584.732769:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005dd56348 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322554432:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732813:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322554176:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732827:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322554496:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315584.732899:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322554112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.732930:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315584.732950:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ddb024e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322554368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.733003:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554112 00000100:00100000:0.0:1761315584.733022:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315584.733032:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686 00000100:00100000:1.0:1761315584.733037:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322554112:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.733054:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322554048:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315584.733074:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322554112:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (143us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315584.733078:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322554112:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.733084:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686 00000100:00100000:0.0:1761315584.733109:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554368 00000100:00100000:0.0:1761315584.733134:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2849 00000100:00100000:0.0:1761315584.733139:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ac61232 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322554368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315584.733188:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ac61232 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322554368:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (166us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315584.733194:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2849 00000100:00100000:3.0:1761315584.733516:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c8817063 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322554304:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315584.733568:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3827 00000100:00080000:2.0:1761315584.733571:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315584.733575:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315584.733598:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001345af7d x1846869322554560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315584.733658:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001345af7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322554560:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315584.733687:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315584.734420:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554560 00000100:00100000:3.0:1761315584.734461:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2238 00000100:00100000:3.0:1761315584.734467:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322554560:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315584.734553:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315584.734672:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322554624:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.734700:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060e3fc1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322554752:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315584.734708:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315584.734737:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315584.734745:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554624 00000100:00100000:2.0:1761315584.734774:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4553 00000100:00100000:2.0:1761315584.734778:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003741f711 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322554624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.735071:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ddb024e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322554368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315584.735087:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315584.735092:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000059c6fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322554688:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315584.735180:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003741f711 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322554624:12345-0@lo:400:kworker/u8:0.0 Request processed in 400us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315584.735187:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4553 00000100:00100000:2.0:1761315584.735218:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322554624:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315584.735647:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322554560:12345-0@lo:8:kworker/u8:0.0 Request processed in 1178us (1958us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315584.735655:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2238 00000100:00100000:3.0:1761315584.735669:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554752 00000100:00100000:3.0:1761315584.735689:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1497 00000100:00100000:3.0:1761315584.735693:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322554752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315584.735724:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322554752:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (988us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315584.735729:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1497 00000100:00080000:0.0:1761315584.735796:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315584.735801:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315584.735805:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001345af7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322554560:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315584.736304:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315584.736337:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060e3fc1e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322554752:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315584.736507:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554688 00000100:00100000:2.0:1761315584.736546:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4554 00000100:00100000:2.0:1761315584.736551:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322554688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315584.736598:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f896a8c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322554688:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315584.736605:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4554 00000100:00100000:1.0:1761315584.736714:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000059c6fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322554688:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315585.755817:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315586.761679:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315586.761714:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315586.762784:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556160 00000100:00100000:2.0:1761315586.764664:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2239 00000100:00100000:2.0:1761315586.764674:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307556160:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315586.764773:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315586.764856:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307556160:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 182us (3183us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315586.764863:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2239 00000100:00100000:1.0:1761315586.765525:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315586.765572:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315586.765915:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307555968 00000100:00100000:0.0:1761315586.766135:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556032 00000100:00100000:0.0:1761315586.766189:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4555 00000100:00100000:0.0:1761315586.766196:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315586.766255:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d01b1820 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 59us (4540us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315586.766261:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4555 00000100:00100000:2.0:1761315586.767003:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1687 00000100:00100000:2.0:1761315586.767010:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307555968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315586.767052:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307555968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (1528us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315586.767059:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1687 00000100:00100000:0.0:1761315586.770743:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556096 00000100:00100000:0.0:1761315586.770813:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4556 00000100:00100000:0.0:1761315586.770819:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307556096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315586.770892:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a08ec369 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307556096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (5321us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315586.770900:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4556 00010000:00010000:0.0:1761315586.779786:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315587.292259:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315587.292280:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322554816:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315587.292365:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315587.292421:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554816 00000100:00100000:3.0:1761315587.292462:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2850 00000100:00100000:3.0:1761315587.292467:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322554816:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315587.292497:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315587.292534:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322554816:12345-0@lo:41:osp-pre-1-0.0 Request processed in 66us (170us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315587.292540:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2850 00000100:00100000:3.0:1761315587.292577:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322554816:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315587.803933:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315588.828067:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315589.852425:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322554880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315589.852496:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315589.852598:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554880 00000100:00100000:3.0:1761315589.852651:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 994 00000100:00100000:3.0:1761315589.852657:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322554880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315589.852686:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315589.852727:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322554880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (231us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315589.852734:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 994 00000100:00100000:1.0:1761315589.852820:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322554880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315589.853576:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322554944:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315589.853615:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315589.853698:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322554944 00000100:00100000:1.0:1761315589.853735:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2851 00000100:00100000:1.0:1761315589.853739:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322554944:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315589.853759:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315589.853790:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322554944:12345-0@lo:41:osp-pre-0-1.0 Request processed in 51us (175us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315589.853795:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2851 00000100:00100000:2.0:1761315589.853854:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322554944:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315589.854153:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315589.854159:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315589.854162:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315589.854167:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3832 00000100:00080000:0.0:1761315589.854169:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315589.854173:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315589.854191:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001eebd285 x1846869322555264/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315589.854206:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315589.854209:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315589.854210:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315589.854213:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3832 00000100:00080000:0.0:1761315589.854214:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315589.854216:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315589.854226:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a4967bd3 x1846869322555328/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315589.854291:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322555264:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854322:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315589.854339:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322555328:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854384:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315589.854408:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315589.854421:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555264 00000100:00100000:0.0:1761315589.854442:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555328 00000100:00100000:0.0:1761315589.854454:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2240 00000100:00100000:0.0:1761315589.854458:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322555264:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315589.854550:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315589.854582:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322555264:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (259us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315589.854588:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2240 00000100:00100000:0.0:1761315589.854600:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2241 00000100:00100000:0.0:1761315589.854603:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049a288df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322555328:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315589.854677:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315589.854702:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049a288df pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322555328:12345-0@lo:8:kworker/u8:2.0 Request processed in 98us (318us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315589.854707:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2241 00000100:00080000:0.0:1761315589.854827:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315589.854831:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315589.854834:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322555264:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315589.854843:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315589.854845:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315589.854847:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a4967bd3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322555328:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854866:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322555136:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854892:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315589.854923:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322555520:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854945:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315589.854970:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315589.854976:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322555072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.854996:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315589.855008:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322555456:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855025:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315589.855051:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555136 00000100:00100000:0.0:1761315589.855077:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555072 00000100:00100000:0.0:1761315589.855090:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555456 00000100:00100000:0.0:1761315589.855103:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4557 00000100:00100000:0.0:1761315589.855106:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322555136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855141:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4967bd3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322555136:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315589.855147:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4557 00000100:00100000:0.0:1761315589.855158:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4558 00000100:00100000:0.0:1761315589.855162:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322555072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855185:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000479f47a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322555072:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (189us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315589.855190:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4558 00000100:00100000:0.0:1761315589.855199:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4559 00000100:00100000:0.0:1761315589.855202:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2bd8cd7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322555456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855226:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2bd8cd7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322555456:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (200us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315589.855230:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4559 00000100:00100000:0.0:1761315589.855246:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555520 00000100:00100000:0.0:1761315589.855265:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1498 00000100:00100000:0.0:1761315589.855268:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322555520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855294:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322555520:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (348us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315589.855299:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1498 00000100:00100000:0.0:1761315589.855330:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322555072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855338:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322555456:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855373:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322555136:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.855380:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322555520:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315589.855472:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322555008:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315589.855505:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315589.855529:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322555392:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315589.855550:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315589.855565:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315589.855570:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322555200:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315589.855591:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315589.855672:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555392 00000100:00100000:2.0:1761315589.855708:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4560 00000100:00100000:2.0:1761315589.855711:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322555392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315589.855747:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cdfd94f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322555392:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (196us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315589.855753:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4560 00000100:00100000:2.0:1761315589.855799:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322555392:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315589.855819:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555200 00000100:00100000:2.0:1761315589.855856:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2852 00000100:00100000:2.0:1761315589.855860:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322555200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315589.855895:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322555200:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (304us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315589.855900:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2852 00000100:00100000:2.0:1761315589.855921:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0f192fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322555200:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.856064:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555008 00000100:00100000:0.0:1761315589.856185:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1688 00000100:00100000:0.0:1761315589.856190:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322555008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315589.856227:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322555008:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (722us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315589.856233:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1688 00000100:00100000:2.0:1761315589.856240:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322555008:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315590.859040:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315590.859100:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315590.859211:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556224 00000100:00100000:2.0:1761315590.859269:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556288 00000100:00100000:2.0:1761315590.860000:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1370 00000100:00100000:2.0:1761315590.860009:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8888b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307556224:12345-192.168.203.51@tcp:17:kworker/1:0.0 00000100:00100000:2.0:1761315590.860108:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8888b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307556224:12345-192.168.203.51@tcp:17:kworker/1:0.0 Request processed in 100us (1076us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315590.860116:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1370 00000100:00100000:2.0:1761315590.860133:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1371 00000100:00100000:2.0:1761315590.860137:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556288:12345-192.168.203.51@tcp:17:kworker/1:0.0 00000100:00100000:2.0:1761315590.860168:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000009cdcdea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556288:12345-192.168.203.51@tcp:17:kworker/1:0.0 Request processed in 30us (1068us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315590.860174:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1371 00010000:00010000:0.0:1761315590.875513:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315591.880257:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315591.880298:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315591.880438:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556416 00000100:00100000:2.0:1761315591.880447:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556352 00000100:00100000:2.0:1761315591.880527:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1689 00000100:00100000:2.0:1761315591.880535:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307556352:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315591.880546:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4561 00000100:00100000:1.0:1761315591.880556:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556416:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315591.880612:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307556352:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (360us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315591.880622:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1689 00000100:00100000:1.0:1761315591.880636:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000793073e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307556416:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 79us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315591.880650:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4561 00000100:00100000:3.0:1761315591.880714:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315591.880878:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315591.881147:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556544 00000100:00100000:2.0:1761315591.881620:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556480 00000100:00100000:2.0:1761315591.881663:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4562 00000100:00100000:2.0:1761315591.881668:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307556480:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315591.881717:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307556480:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 49us (1010us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315591.881724:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4562 00000100:00100000:0.0:1761315591.881806:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2242 00000100:00100000:0.0:1761315591.881813:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307556544:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315591.881911:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315591.881993:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307556544:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 178us (1115us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315591.882002:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2242 00010000:00010000:1.0:1761315591.900299:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315592.412072:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322555584:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315592.412170:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315592.412714:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555584 00000100:00100000:1.0:1761315592.412779:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2853 00000100:00100000:1.0:1761315592.412786:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322555584:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315592.412831:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315592.412898:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322555584:12345-0@lo:41:osp-pre-1-0.0 Request processed in 111us (729us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315592.412907:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2853 00000100:00100000:3.0:1761315592.413011:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322555584:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315592.926183:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315593.947533:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315594.972082:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315594.972179:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315594.972244:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315594.972449:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322555712:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315594.972459:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315594.972464:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315594.972468:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315594.972474:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3837 00000100:00080000:3.0:1761315594.972477:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315594.972482:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315594.972487:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555648 00000100:00100000:2.0:1761315594.972491:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315594.972502:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bb94dc5e x1846869322556096/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315594.972518:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322555968:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315594.972520:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315594.972526:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315594.972528:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315594.972532:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3837 00000100:00100000:1.0:1761315594.972535:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 995 00000100:00080000:3.0:1761315594.972536:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315594.972540:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315594.972547:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000178f802 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322555648:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315594.972554:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315594.972554:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d0ae6c25 x1846869322556160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315594.972579:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00002000:00000024:1.0:1761315594.972581:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315594.972590:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322556352:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315594.972601:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322556096:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315594.972620:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315594.972639:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000178f802 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322555648:12345-0@lo:13:osp-pre-1-1.0 Request processed in 91us (394us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315594.972649:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322555776:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315594.972650:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 995 00000100:00100000:3.0:1761315594.972651:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315594.972677:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322556160:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315594.972680:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555968 00000100:00100000:2.0:1761315594.972683:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315594.972708:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322556032:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315594.972709:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555776 00000100:00100000:3.0:1761315594.972712:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315594.972734:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4563 00000100:00100000:1.0:1761315594.972739:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ca269b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322555968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.972745:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315594.972754:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556352 00000100:00100000:3.0:1761315594.972787:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556096 00000100:00100000:2.0:1761315594.972790:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4564 00000100:00100000:2.0:1761315594.972796:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322555776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315594.972801:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556160 00000100:00100000:3.0:1761315594.972815:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1499 00000100:00100000:3.0:1761315594.972820:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322556352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.972839:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008850d91b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322555776:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (155us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315594.972846:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4564 00000100:00100000:3.0:1761315594.972859:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322556352:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315594.972867:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1499 00000100:00100000:2.0:1761315594.972879:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556032 00000100:00100000:1.0:1761315594.972880:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ca269b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322555968:12345-0@lo:400:kworker/u8:2.0 Request processed in 132us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315594.972884:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2243 00000100:00100000:1.0:1761315594.972894:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4563 00000100:00100000:3.0:1761315594.972894:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322556096:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315594.972911:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2854 00000100:00100000:2.0:1761315594.972918:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322556032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315594.972944:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2244 00000100:00100000:1.0:1761315594.972949:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322556160:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315594.972960:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322556032:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (217us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315594.972966:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2854 00000100:00100000:2.0:1761315594.972998:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322556352:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973007:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322555776:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973013:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322556032:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973020:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee31279c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322555968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973059:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006047b3cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555904:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315594.973079:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315594.973090:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322556288:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973105:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315594.973124:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555648:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315594.973134:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761315594.973138:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555840:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973153:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315594.973165:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322556224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973192:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315594.973210:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315594.973211:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555904 00000100:00100000:2.0:1761315594.973232:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555840 00000100:00100000:2.0:1761315594.973245:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2855 00000100:00100000:2.0:1761315594.973249:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322555904:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315594.973254:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322556160:12345-0@lo:8:kworker/u8:2.0 Request processed in 302us (545us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315594.973261:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2244 00000004:00000024:2.0:1761315594.973274:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315594.973284:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2856 00000100:00100000:1.0:1761315594.973290:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322555840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973311:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322555904:12345-0@lo:41:osp-pre-0-1.0 Request processed in 60us (230us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315594.973317:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2855 00000100:00100000:2.0:1761315594.973364:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006047b3cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555904:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315594.973427:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322555712 00010000:02020000:3.0:1761315594.973438:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315594.973448:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1690 00000100:00100000:2.0:1761315594.973452:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322555712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315594.973466:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322556096:12345-0@lo:8:kworker/u8:2.0 Request processed in 574us (822us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315594.973474:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2243 00000100:00100000:2.0:1761315594.973489:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322555712:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (1000us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315594.973499:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1690 00000100:00100000:3.0:1761315594.973525:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322555712:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315594.973540:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556288 00000100:00100000:3.0:1761315594.973570:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556224 00000100:00100000:3.0:1761315594.973596:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4565 00000100:00100000:3.0:1761315594.973600:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060e3fc1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322556288:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315594.973624:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315594.973629:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315594.973634:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322556096:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315594.973647:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060e3fc1e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322556288:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (539us total) trans 0 rc 0/0 00000100:00080000:0.0:1761315594.973648:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315594.973652:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315594.973656:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4565 00000100:00100000:0.0:1761315594.973657:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322556160:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315594.973675:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4566 00000100:00100000:3.0:1761315594.973679:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ddb024e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322556224:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315594.973706:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ddb024e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322556224:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (513us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315594.973712:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4566 00000100:00100000:0.0:1761315594.973721:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322556224:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315594.973732:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322556288:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315594.973963:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322555840:12345-0@lo:400:kworker/u8:2.0 Request processed in 673us (809us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315594.973970:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2856 00000100:00100000:0.0:1761315594.974653:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322555840:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315595.995867:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315597.000325:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315597.000405:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315597.000445:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315597.000450:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556608 00000100:00100000:2.0:1761315597.000578:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1691 00000100:00100000:2.0:1761315597.000586:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307556608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315597.000659:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307556608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (342us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315597.000666:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1691 00000100:00100000:1.0:1761315597.001572:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315597.001809:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556736 00000100:00100000:2.0:1761315597.001810:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556800 00000100:00100000:3.0:1761315597.001850:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556672 00000100:00100000:2.0:1761315597.001858:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2245 00000100:00100000:2.0:1761315597.001864:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307556800:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761315597.001877:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4567 00000100:00100000:3.0:1761315597.001883:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000977da04 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307556736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315597.001964:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000977da04 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307556736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 81us (1560us total) trans 0 rc 0/0 00010000:02020000:2.0:1761315597.001974:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315597.001974:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4567 00000100:00100000:3.0:1761315597.001993:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4568 00000100:00100000:3.0:1761315597.001998:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2fff388 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307556672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315597.002017:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307556800:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 150us (446us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315597.002024:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2245 00000100:00100000:3.0:1761315597.002029:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2fff388 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307556672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 29us (1583us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315597.002036:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4568 00010000:00010000:0.0:1761315597.020142:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315597.531553:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322556416:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315597.531624:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315597.531759:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556416 00000100:00100000:1.0:1761315597.531804:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2857 00000100:00100000:1.0:1761315597.531811:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322556416:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315597.531841:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315597.531886:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322556416:12345-0@lo:41:osp-pre-1-0.0 Request processed in 75us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315597.531892:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2857 00000100:00100000:0.0:1761315597.531951:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322556416:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315598.043569:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315599.068274:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315600.091579:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315600.091587:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315600.091590:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315600.091597:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3842 00000100:00080000:3.0:1761315600.091600:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315600.091606:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315600.091624:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007245de9d x1846869322556800/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315600.091636:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315600.091638:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315600.091639:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315600.091642:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3842 00000100:00080000:3.0:1761315600.091644:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315600.091646:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315600.091654:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d0ae6c25 x1846869322556864/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315600.091993:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322556544:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092060:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092088:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322556928:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092109:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092131:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1761315600.092136:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322556480:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092162:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092174:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322556736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092194:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092209:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322557120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092231:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092253:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322556608:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092274:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092289:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322556992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092309:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092321:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315600.092326:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322556672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092371:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092397:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322557056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092418:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.092448:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556544 00000100:00100000:1.0:1761315600.092482:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556928 00000100:00100000:1.0:1761315600.092495:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556992 00000100:00100000:1.0:1761315600.092505:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556672 00000100:00100000:1.0:1761315600.092517:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4569 00000100:00100000:1.0:1761315600.092522:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322556544:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092565:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047f2d143 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322556544:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (506us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092572:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4569 00000100:00100000:1.0:1761315600.092586:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4570 00000100:00100000:1.0:1761315600.092589:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322556928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092594:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4571 00000100:00100000:3.0:1761315600.092599:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322556992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092613:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd62b949 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322556928:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (504us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092619:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4570 00000100:00100000:1.0:1761315600.092629:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4572 00000100:00100000:3.0:1761315600.092629:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa32dc4d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322556992:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (320us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092632:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322556672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092634:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4571 00000100:00100000:1.0:1761315600.092655:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085790e43 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322556672:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092660:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4572 00000100:00100000:3.0:1761315600.092664:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322556992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092675:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556736 00000100:00100000:3.0:1761315600.092682:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322556672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092693:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556608 00000100:00100000:1.0:1761315600.092704:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2858 00000100:00100000:1.0:1761315600.092707:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322556736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092732:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322556736:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (537us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092737:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2858 00000100:00100000:1.0:1761315600.092746:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2859 00000100:00100000:1.0:1761315600.092749:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322556608:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092768:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557120 00000100:00100000:1.0:1761315600.092775:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322556608:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (501us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092779:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2859 00000100:00100000:3.0:1761315600.092788:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557056 00000100:00100000:1.0:1761315600.092799:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322556608:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092800:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1500 00000100:00100000:3.0:1761315600.092803:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322557120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092818:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322556736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092826:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322556544:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092829:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322557120:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (599us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.092833:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322556928:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092835:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1500 00000100:00100000:3.0:1761315600.092844:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1501 00000100:00100000:3.0:1761315600.092847:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322557056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.092850:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322557120:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.092879:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322557056:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (460us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315600.092894:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1501 00000100:00100000:1.0:1761315600.092910:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322557056:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315600.095428:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322556800:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315600.095491:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315600.095517:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322556864:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315600.095543:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315600.095609:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556800 00000100:00100000:3.0:1761315600.095632:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556864 00000100:00100000:3.0:1761315600.095644:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2246 00000100:00100000:3.0:1761315600.095649:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322556800:12345-0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315600.095674:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:02020000:3.0:1761315600.095743:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315600.095792:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322556800:12345-0@lo:8:kworker/u8:2.0 Request processed in 142us (302us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315600.095799:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2246 00000100:00100000:3.0:1761315600.095814:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2247 00000100:00100000:3.0:1761315600.095817:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322556864:12345-0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315600.095905:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00010000:02020000:3.0:1761315600.095908:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1761315600.095910:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315600.095916:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322556800:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315600.095936:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322556864:12345-0@lo:8:kworker/u8:2.0 Request processed in 116us (392us total) trans 0 rc -19/-19 00000100:00080000:0.0:1761315600.095943:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315600.095946:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315600.095950:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322556864:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315600.095959:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2247 00000100:00100000:2.0:1761315600.101411:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322556480 00000100:00100000:2.0:1761315600.101481:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692 00000100:00100000:2.0:1761315600.101487:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322556480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.101501:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315600.101513:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322557184:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315600.101548:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315600.101558:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322556480:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (9396us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315600.101570:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692 00000100:00100000:1.0:1761315600.101576:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557184 00000100:00100000:1.0:1761315600.101596:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2860 00000100:00100000:1.0:1761315600.101599:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322557184:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315600.101620:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315600.101661:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322557184:12345-0@lo:41:osp-pre-0-1.0 Request processed in 61us (113us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315600.101668:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2860 00000100:00100000:1.0:1761315600.101697:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322557184:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315600.101710:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315600.101718:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322557248:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315600.101753:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315600.101847:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557248 00000100:00100000:1.0:1761315600.101891:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 996 00000100:00100000:2.0:1761315600.101894:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322556480:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315600.101896:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322557248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315600.101913:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315600.102009:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322557248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 112us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315600.102014:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322557248:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315600.102015:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 996 00010000:00010000:1.0:1761315601.119069:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315602.127016:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315602.134835:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315602.134898:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315602.134925:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315602.134995:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557056 00000100:00100000:0.0:1761315602.134996:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556864 00000100:00100000:3.0:1761315602.135044:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2248 00000100:00100000:3.0:1761315602.135050:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307557056:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761315602.135182:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1693 00000100:00100000:0.0:1761315602.135188:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307556864:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315602.135229:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556928 00000100:00100000:0.0:1761315602.135260:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307556864:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 70us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315602.135271:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1693 00000100:00100000:1.0:1761315602.135272:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4573 00000100:00100000:1.0:1761315602.135279:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8817063 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307556928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315602.135316:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8817063 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307556928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 37us (8308us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315602.135322:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4573 00010000:02020000:3.0:1761315602.135667:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315602.135710:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307557056:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 659us (882us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315602.135718:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2248 00000100:00100000:1.0:1761315602.136472:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307556992 00000100:00100000:1.0:1761315602.136512:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4574 00000100:00100000:1.0:1761315602.136517:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007cf56814 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307556992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315602.136569:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007cf56814 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307556992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 51us (1644us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315602.136576:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4574 00010000:00010000:0.0:1761315602.139944:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315602.651588:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d653401f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322557312:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315602.651666:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315602.651763:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557312 00000100:00100000:1.0:1761315602.651815:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2861 00000100:00100000:1.0:1761315602.651820:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322557312:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315602.651849:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315602.653844:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322557312:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2016us (2172us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315602.653858:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2861 00000100:00100000:0.0:1761315602.660041:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d653401f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322557312:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315603.164170:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315604.188526:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315605.211582:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315605.211607:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315605.211627:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322557440:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315605.211691:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315605.211734:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322557376:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315605.211756:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315605.211880:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557440 00000100:00100000:3.0:1761315605.211933:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2862 00000100:00100000:3.0:1761315605.211940:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322557440:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315605.211970:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315605.212012:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322557440:12345-0@lo:41:osp-pre-0-1.0 Request processed in 73us (322us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315605.212018:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2862 00000100:00100000:3.0:1761315605.212057:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322557440:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315605.212167:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557376 00000100:00100000:2.0:1761315605.212197:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 997 00000100:00100000:2.0:1761315605.212201:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322557376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315605.212227:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315605.212237:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315605.212259:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322557376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 57us (502us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315605.212265:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 997 00000100:00100000:2.0:1761315605.212522:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322557568:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315605.212536:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000883760dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322557696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212555:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315605.212577:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315605.212581:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315605.212588:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000825eb04a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322557632:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315605.212597:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322557504:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212618:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315605.212629:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315605.212654:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557568 00000100:00100000:0.0:1761315605.212667:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557696 00000100:00100000:2.0:1761315605.212689:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557632 00000100:00100000:0.0:1761315605.212694:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2863 00000100:00100000:0.0:1761315605.212699:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042bf6d90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322557696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212713:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4575 00000100:00100000:2.0:1761315605.212718:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e24bfdb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322557568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315605.212734:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042bf6d90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322557696:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (158us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315605.212741:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2863 00000100:00100000:2.0:1761315605.212757:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e24bfdb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322557568:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (202us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315605.212763:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4575 00000100:00100000:0.0:1761315605.212769:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000883760dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322557696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212780:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4576 00000100:00100000:2.0:1761315605.212785:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007520ddc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322557632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212807:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007520ddc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322557632:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (192us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315605.212815:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4576 00000100:00100000:2.0:1761315605.212838:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000825eb04a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322557632:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212845:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004d0d8c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322557568:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315605.212857:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315605.212863:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315605.212866:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315605.212872:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3847 00000100:00080000:1.0:1761315605.212874:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315605.212878:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315605.212897:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000059ff679c x1846869322557760/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315605.212918:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315605.212921:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315605.212922:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315605.212925:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3847 00000100:00080000:1.0:1761315605.212927:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315605.212930:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315605.212939:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059ff679c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322557760:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315605.212965:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315605.212992:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557760 00000100:00100000:2.0:1761315605.213008:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2249 00000100:00100000:2.0:1761315605.213011:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322557760:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315605.213090:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315605.213115:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322557760:12345-0@lo:8:kworker/u8:0.0 Request processed in 104us (150us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315605.213120:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2249 00000100:00080000:2.0:1761315605.213139:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315605.213142:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315605.213145:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059ff679c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322557760:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214137:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007b0c2887 x1846869322557824/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315605.214199:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315605.214323:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322557824:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214335:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322558016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214397:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315605.214419:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322557952:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214445:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558016 00000100:00100000:2.0:1761315605.214450:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315605.214485:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1502 00000100:00100000:1.0:1761315605.214491:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322558016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.214491:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557952 00000100:00100000:2.0:1761315605.214526:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4577 00000100:00100000:2.0:1761315605.214532:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005dd56348 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322557952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214538:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322558016:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315605.214545:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1502 00000100:00100000:1.0:1761315605.214577:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322558016:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315605.214582:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005dd56348 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322557952:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (133us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315605.214591:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4577 00000100:00100000:1.0:1761315605.214603:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315605.214629:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322557952:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315605.214635:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557824 00000100:00100000:1.0:1761315605.214655:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2250 00000100:00100000:1.0:1761315605.214659:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322557824:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315605.214742:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315605.214781:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322557824:12345-0@lo:8:kworker/u8:0.0 Request processed in 121us (180us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315605.214787:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2250 00000100:00100000:0.0:1761315605.214858:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000392f96aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322557888:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315605.214886:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315605.214981:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557504 00000100:00100000:3.0:1761315605.215014:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1694 00000100:00100000:3.0:1761315605.215018:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322557504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315605.215053:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322557504:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (2425us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315605.215060:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1694 00000100:00100000:3.0:1761315605.215105:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322557376:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315605.215169:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005522250c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322557504:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315605.215283:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315605.215287:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315605.215291:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322557824:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315605.220902:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322557888 00000100:00100000:1.0:1761315605.220974:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4578 00000100:00100000:1.0:1761315605.220981:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f9a6d13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322557888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315605.221077:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f9a6d13 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322557888:12345-0@lo:400:kworker/u8:0.0 Request processed in 96us (6191us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315605.221088:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4578 00000100:00100000:0.0:1761315605.221186:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000392f96aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322557888:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315606.235514:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315607.241658:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315607.241754:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557120 00000100:00100000:1.0:1761315607.241803:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1695 00000100:00100000:1.0:1761315607.241809:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307557120:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315607.241906:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307557120:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 96us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315607.241913:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1695 00000100:00100000:3.0:1761315607.243153:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315607.243189:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315607.243253:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557184 00000100:00100000:1.0:1761315607.243295:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557248 00000100:00100000:1.0:1761315607.243324:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4579 00000100:00100000:1.0:1761315607.243329:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006f75a3eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557184:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315607.243407:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006f75a3eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557184:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 77us (256us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315607.243414:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4579 00000100:00100000:1.0:1761315607.243427:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4580 00000100:00100000:1.0:1761315607.243430:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d653401f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315607.243454:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d653401f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315607.243460:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4580 00000100:00100000:2.0:1761315607.244538:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315607.244613:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557312 00000100:00100000:1.0:1761315607.244645:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2251 00000100:00100000:1.0:1761315607.244650:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307557312:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315607.244742:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315607.244795:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307557312:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 143us (260us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315607.244801:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2251 00010000:00010000:1.0:1761315607.259677:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315607.773829:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315607.773859:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558080:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315607.773924:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315607.774015:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558080 00000100:00100000:3.0:1761315607.774063:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2864 00000100:00100000:3.0:1761315607.774069:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2000c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558080:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315607.774099:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315607.774140:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2000c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558080:12345-0@lo:41:osp-pre-1-0.0 Request processed in 70us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315607.774147:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2864 00000100:00100000:1.0:1761315607.775811:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558080:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315608.283536:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315609.307530:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315610.331779:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315610.332195:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315610.332200:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315610.332203:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315610.332211:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3852 00000100:00080000:3.0:1761315610.332214:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315610.332219:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315610.332238:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bb94dc5e x1846869322558528/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315610.332260:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315610.332263:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315610.332264:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315610.332267:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3852 00000100:00080000:3.0:1761315610.332269:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315610.332272:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315610.332283:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000137dd527 x1846869322558592/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315610.332303:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322558528:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315610.332386:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315610.332410:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322558592:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315610.332432:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315610.332461:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322558144:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.332477:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558528 00000100:00100000:2.0:1761315610.332500:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322558400:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315610.332508:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315610.332510:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558592 00000100:00100000:0.0:1761315610.332530:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2252 00000100:00100000:1.0:1761315610.332535:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322558208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315610.332538:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322558528:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332540:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315610.332562:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322558784:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315610.332572:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315610.332587:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315610.332597:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558464:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332621:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322558272:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315610.332627:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:0.0:1761315610.332644:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315610.332651:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558848:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332654:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315610.332675:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322558656:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.332688:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322558528:12345-0@lo:8:kworker/u8:2.0 Request processed in 151us (302us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315610.332697:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2252 00000100:00100000:1.0:1761315610.332698:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315610.332702:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315610.332716:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558784 00000100:00100000:2.0:1761315610.332723:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ac28ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322558912:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315610.332736:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558848 00000100:00100000:1.0:1761315610.332743:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322558336:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332745:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315610.332748:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1503 00000100:00100000:0.0:1761315610.332753:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322558784:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332780:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1504 00000100:00100000:1.0:1761315610.332785:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315610.332788:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab5a3502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558848:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.332795:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322558784:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (210us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315610.332801:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1503 00000100:00100000:0.0:1761315610.332826:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2253 00000100:00100000:0.0:1761315610.332830:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322558592:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332832:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab5a3502 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558848:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (139us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315610.332837:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1504 00000100:00100000:1.0:1761315610.332855:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322558720:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.332863:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558400 00000100:00100000:1.0:1761315610.332884:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:0.0:1761315610.332909:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315610.332925:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558144 00000100:00100000:0.0:1761315610.332938:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322558592:12345-0@lo:8:kworker/u8:2.0 Request processed in 107us (505us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315610.332945:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2253 00000100:00100000:1.0:1761315610.332957:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696 00000100:00100000:1.0:1761315610.332963:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322558144:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315610.332977:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315610.332981:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315610.332986:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322558528:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315610.332999:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315610.333002:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315610.333003:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322558144:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (496us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315610.333007:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322558592:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315610.333014:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696 00000100:00100000:3.0:1761315610.333025:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558464 00000100:00100000:3.0:1761315610.333054:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558912 00000100:00100000:0.0:1761315610.333056:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558336 00000100:00100000:0.0:1761315610.333069:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2865 00000100:00100000:0.0:1761315610.333073:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558464:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315610.333075:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2866 00000100:00100000:3.0:1761315610.333078:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322558336:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.333101:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322558464:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (476us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315610.333105:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322558336:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (324us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315610.333107:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2865 00000100:00100000:3.0:1761315610.333109:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2866 00000100:00100000:3.0:1761315610.333119:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2867 00000100:00100000:3.0:1761315610.333121:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322558912:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315610.333133:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558208 00000004:00000024:3.0:1761315610.333140:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315610.333152:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 998 00000100:00100000:0.0:1761315610.333156:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322558208:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315610.333167:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322558912:12345-0@lo:41:osp-pre-0-1.0 Request processed in 45us (425us total) trans 0 rc 0/0 00002000:00000024:0.0:1761315610.333172:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315610.333172:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2867 00000100:00100000:0.0:1761315610.333194:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322558208:12345-0@lo:13:osp-pre-1-1.0 Request processed in 37us (624us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315610.333200:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 998 00000100:00100000:0.0:1761315610.333224:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322558144:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.333237:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8e92ee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322558208:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315610.333245:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315610.333253:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3fb549f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322558848:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.335810:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558272 00000100:00100000:2.0:1761315610.335882:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558656 00000100:00100000:2.0:1761315610.335892:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558720 00000100:00100000:2.0:1761315610.335904:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4581 00000100:00100000:2.0:1761315610.335910:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322558400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.335979:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027585be3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322558400:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (3443us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315610.335985:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4581 00000100:00100000:2.0:1761315610.335998:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4582 00000100:00100000:2.0:1761315610.336001:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005522250c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322558272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336019:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005522250c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322558272:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (3369us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315610.336023:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4582 00000100:00100000:2.0:1761315610.336031:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4583 00000100:00100000:2.0:1761315610.336033:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d0d8c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322558656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336049:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d0d8c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322558656:12345-0@lo:400:kworker/u8:2.0 Request processed in 15us (3348us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315610.336065:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4583 00000100:00100000:2.0:1761315610.336073:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4584 00000100:00100000:2.0:1761315610.336075:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007adea527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322558720:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336094:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007adea527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322558720:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (3214us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315610.336099:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4584 00000100:00100000:2.0:1761315610.336159:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007245de9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322558400:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336173:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d2dc936e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322558784:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336214:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b600536 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322558272:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336223:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034d7daff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322558656:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336238:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ac28ac7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322558912:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315610.336296:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322558336:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315610.336305:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322558720:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315611.356225:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315612.362519:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315612.363714:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557376 00000100:00100000:1.0:1761315612.363781:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1697 00000100:00100000:1.0:1761315612.363788:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307557376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315612.364840:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307557376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1048us (2497us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315612.364852:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1697 00000100:00100000:0.0:1761315612.374617:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315612.374644:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315612.374686:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315612.374734:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557504 00000100:00100000:3.0:1761315612.374790:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557440 00000100:00100000:3.0:1761315612.374823:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4585 00000100:00100000:3.0:1761315612.374828:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315612.374902:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8e92ee1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315612.374910:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4585 00000100:00100000:1.0:1761315612.374924:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4586 00000100:00100000:1.0:1761315612.374929:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315612.374965:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0f192fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (321us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315612.375141:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557568 00000100:00100000:2.0:1761315612.375167:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2254 00000100:00100000:2.0:1761315612.375171:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307557568:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315612.375260:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315612.375289:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307557568:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 118us (604us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315612.375295:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2254 00000100:00100000:1.0:1761315612.378555:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4586 00010000:00010000:0.0:1761315612.379587:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315612.893586:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315612.894198:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322558976:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315612.894263:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315612.894374:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322558976 00000100:00100000:2.0:1761315612.894548:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2868 00000100:00100000:2.0:1761315612.894554:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322558976:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315612.894585:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315612.894955:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322558976:12345-0@lo:41:osp-pre-1-0.0 Request processed in 400us (692us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315612.894963:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2868 00000100:00100000:1.0:1761315612.895028:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322558976:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315613.404159:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315614.427776:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315615.451570:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315615.451714:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322559040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315615.451788:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315615.452735:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559040 00000100:00100000:0.0:1761315615.452796:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 999 00000100:00100000:0.0:1761315615.452802:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322559040:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315615.452831:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315615.452875:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322559040:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (1089us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315615.452883:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 999 00000100:00100000:2.0:1761315615.452972:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322559040:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315615.453146:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315615.453154:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322559104:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315615.453183:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315615.453252:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559104 00000100:00100000:3.0:1761315615.453288:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2869 00000100:00100000:3.0:1761315615.453292:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322559104:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315615.453312:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315615.453372:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322559104:12345-0@lo:41:osp-pre-0-1.0 Request processed in 79us (189us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315615.453379:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2869 00000100:00100000:3.0:1761315615.453407:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d0ae6c25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322559104:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315615.453628:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315615.453633:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315615.453636:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315615.453641:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3857 00000100:00080000:0.0:1761315615.453643:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315615.453647:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315615.453673:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e59c3bf1 x1846869322559424/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315615.453687:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315615.453690:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315615.453691:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315615.453694:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3857 00000100:00080000:0.0:1761315615.453695:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761315615.453697:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315615.453708:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f0aeaf6 x1846869322559488/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315615.453740:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e59c3bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322559424:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315615.453766:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315615.453791:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f0aeaf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322559488:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315615.453811:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315615.453837:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559424 00000100:00100000:0.0:1761315615.453853:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559488 00000100:00100000:0.0:1761315615.453867:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2255 00000100:00100000:0.0:1761315615.453870:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d643a28 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322559424:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315615.453957:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315615.453988:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d643a28 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322559424:12345-0@lo:8:kworker/u8:0.0 Request processed in 117us (223us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315615.453994:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2255 00000100:00100000:0.0:1761315615.454005:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2256 00000100:00100000:0.0:1761315615.454008:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322559488:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315615.454087:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315615.454110:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322559488:12345-0@lo:8:kworker/u8:0.0 Request processed in 100us (298us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315615.454115:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2256 00000100:00080000:0.0:1761315615.454137:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315615.454141:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315615.454144:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e59c3bf1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322559424:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315615.454151:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315615.454153:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315615.454155:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f0aeaf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322559488:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454221:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067acb148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322559168:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454251:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315615.454276:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000c58668 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322559552:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454300:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315615.454315:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315615.454320:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b372c039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322559232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454361:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315615.454376:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2541175 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322559616:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454396:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315615.454425:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559552 00000100:00100000:2.0:1761315615.454451:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559232 00000100:00100000:2.0:1761315615.454466:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559616 00000100:00100000:2.0:1761315615.454479:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4587 00000100:00100000:2.0:1761315615.454482:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9965e9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322559552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454515:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9965e9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322559552:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (215us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315615.454521:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4587 00000100:00100000:2.0:1761315615.454534:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4588 00000100:00100000:2.0:1761315615.454537:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322559232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454559:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322559232:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315615.454565:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4588 00000100:00100000:2.0:1761315615.454573:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4589 00000100:00100000:2.0:1761315615.454576:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322559616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454598:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322559616:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (201us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315615.454602:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4589 00000100:00100000:2.0:1761315615.454635:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b372c039 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322559232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454644:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2541175 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322559616:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315615.454652:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000c58668 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322559552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.454805:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000087b042 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322559296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.454831:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315615.454856:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c91750f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322559680:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.454875:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315615.454892:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315615.454897:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002deaa2ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322559360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.454918:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315615.454941:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559680 00000100:00100000:3.0:1761315615.454965:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1505 00000100:00100000:3.0:1761315615.454969:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322559680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.455001:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322559680:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (125us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315615.455006:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1505 00000100:00100000:3.0:1761315615.455024:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559360 00000100:00100000:3.0:1761315615.455044:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2870 00000100:00100000:3.0:1761315615.455048:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322559360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.455074:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322559360:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (157us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315615.455080:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2870 00000100:00100000:3.0:1761315615.455103:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002deaa2ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322559360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315615.455110:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c91750f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322559680:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315615.455200:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559296 00000100:00100000:1.0:1761315615.455234:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4590 00000100:00100000:1.0:1761315615.455237:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322559296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315615.455276:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0ae6c25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322559296:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (444us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315615.455282:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4590 00000100:00100000:1.0:1761315615.455302:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000087b042 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322559296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315615.455701:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559168 00000100:00100000:0.0:1761315615.455736:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698 00000100:00100000:0.0:1761315615.455740:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322559168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315615.455792:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322559168:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (1541us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315615.455798:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698 00000100:00100000:3.0:1761315615.455864:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067acb148 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322559168:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315616.476411:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315617.483383:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315617.485063:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315617.486773:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315617.486815:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557696 00000100:00100000:2.0:1761315617.487169:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4591 00000100:00100000:2.0:1761315617.487176:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315617.487243:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d2dc936e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307557696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 67us (3905us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315617.487250:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4591 00000100:00100000:1.0:1761315617.487697:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557632 00000100:00100000:1.0:1761315617.487730:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1699 00000100:00100000:1.0:1761315617.487734:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307557632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315617.487775:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307557632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 27us (2704us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315617.487781:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1699 00000100:00100000:3.0:1761315617.489065:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315617.489486:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557824 00000100:00100000:1.0:1761315617.489528:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2257 00000100:00100000:1.0:1761315617.489533:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307557824:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761315617.489600:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557760 00010000:02020000:1.0:1761315617.489631:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315617.489637:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4592 00000100:00100000:0.0:1761315617.489641:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315617.489681:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007245de9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307557760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 40us (2921us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315617.489689:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4592 00000100:00100000:1.0:1761315617.490196:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307557824:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 660us (1132us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315617.490203:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2257 00010000:00010000:0.0:1761315617.499580:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315618.012509:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322559744:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315618.012589:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315618.012689:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559744 00000100:00100000:2.0:1761315618.012752:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2871 00000100:00100000:2.0:1761315618.012767:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322559744:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315618.012801:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315618.014170:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322559744:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1405us (1576us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315618.014182:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2871 00000100:00100000:1.0:1761315618.014400:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322559744:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315618.523603:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315619.548821:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315620.571479:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315620.571498:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322559808:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315620.571560:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315620.571605:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559808 00000100:00100000:2.0:1761315620.571655:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2872 00000100:00100000:2.0:1761315620.571660:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322559808:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315620.571688:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315620.571733:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322559808:12345-0@lo:41:osp-pre-0-1.0 Request processed in 73us (174us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315620.571765:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2872 00000100:00100000:1.0:1761315620.571835:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322559808:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315620.572003:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315620.572202:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322559936:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.572234:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315620.573017:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315620.573024:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322560000:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.573076:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315620.573113:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559936 00000100:00100000:1.0:1761315620.573146:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1700 00000100:00100000:1.0:1761315620.573151:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322559936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.573188:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322559936:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (954us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315620.573193:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1700 00000100:00100000:1.0:1761315620.573224:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560000 00000100:00100000:1.0:1761315620.573250:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4593 00000100:00100000:1.0:1761315620.573253:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c01d5573 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322560000:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.573282:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c01d5573 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322560000:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315620.573287:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4593 00000100:00100000:1.0:1761315620.573317:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322560000:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.573326:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322559936:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315620.574433:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315620.574489:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322560064:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315620.574531:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315620.574558:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f0aeaf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322559872:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315620.574578:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315620.574655:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560064 00000100:00100000:1.0:1761315620.574694:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4594 00000100:00100000:1.0:1761315620.574698:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322560064:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315620.574736:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322560064:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (206us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315620.574753:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4594 00000100:00100000:2.0:1761315620.574823:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322560064:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315620.574885:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322559872 00000100:00100000:0.0:1761315620.574922:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1000 00000100:00100000:0.0:1761315620.574925:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322559872:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315620.574947:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315620.578575:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f0aeaf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322559872:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315620.583597:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322559872:12345-0@lo:13:osp-pre-1-1.0 Request processed in 8662us (9010us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315620.583611:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1000 00000100:00080000:3.0:1761315620.583909:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315620.583914:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315620.583918:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315620.583924:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3863 00000100:00080000:3.0:1761315620.583926:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315620.583930:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315620.583951:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000137dd527 x1846869322560192/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315620.583972:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315620.583975:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315620.583976:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315620.583979:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3863 00000100:00080000:3.0:1761315620.583981:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315620.583983:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315620.583994:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001883266f x1846869322560256/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315620.584076:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322560384:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.584131:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315620.584159:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315620.584164:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322560320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.584185:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315620.584215:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560384 00000100:00100000:3.0:1761315620.584250:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560320 00000100:00100000:3.0:1761315620.584267:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4595 00000100:00100000:3.0:1761315620.584272:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322560384:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.584317:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009400013f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322560384:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (187us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315620.584323:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4595 00000100:00100000:3.0:1761315620.584336:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4596 00000100:00100000:3.0:1761315620.584338:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000249f09c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322560320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.584383:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000249f09c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322560320:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (197us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315620.584388:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4596 00000100:00100000:3.0:1761315620.584425:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322560320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.584434:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322560384:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315620.584532:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322560448:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315620.584563:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315620.584595:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315620.584600:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322560128:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315620.584623:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315620.584695:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322560192:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315620.584720:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315620.584754:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322560256:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315620.584772:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315620.584836:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560448 00000100:00100000:2.0:1761315620.584867:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560192 00000100:00100000:2.0:1761315620.584882:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560256 00000100:00100000:2.0:1761315620.584895:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1506 00000100:00100000:2.0:1761315620.584899:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322560448:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315620.584930:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322560448:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315620.584936:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1506 00000100:00100000:2.0:1761315620.584948:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2258 00000100:00100000:2.0:1761315620.584951:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e59c3bf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322560192:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315620.585040:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315620.585067:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e59c3bf1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322560192:12345-0@lo:8:kworker/u8:0.0 Request processed in 114us (346us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315620.585073:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2258 00000100:00100000:2.0:1761315620.585084:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2259 00000100:00100000:2.0:1761315620.585086:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322560256:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315620.585163:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315620.585181:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322560256:12345-0@lo:8:kworker/u8:0.0 Request processed in 93us (409us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315620.585186:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2259 00000100:00100000:2.0:1761315620.585216:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322560448:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315620.585287:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315620.585291:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315620.585294:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000137dd527 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322560192:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315620.585301:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315620.585303:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315620.585305:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322560256:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315620.585538:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560128 00000100:00100000:3.0:1761315620.585577:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2873 00000100:00100000:3.0:1761315620.585582:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322560128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315620.585628:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322560128:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1005us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315620.585634:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2873 00000100:00100000:0.0:1761315620.585703:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006a8ab968 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322560128:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315621.599128:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315622.604059:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315622.604101:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315622.604142:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315622.604219:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557888 00000100:00100000:3.0:1761315622.604236:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558080 00000100:00100000:1.0:1761315622.604274:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1701 00000100:00100000:1.0:1761315622.604282:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307557888:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315622.604284:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2260 00000100:00100000:3.0:1761315622.604295:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307558080:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:1.0:1761315622.604404:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307557888:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 123us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315622.604414:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1701 00010000:02020000:3.0:1761315622.604420:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315622.604456:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307558080:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 164us (406us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315622.604463:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2260 00000100:00100000:2.0:1761315622.604585:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307557952 00000100:00100000:2.0:1761315622.604624:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4597 00000100:00100000:2.0:1761315622.604629:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a3ff52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307557952:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315622.604789:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a3ff52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307557952:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 158us (687us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315622.604796:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4597 00000100:00100000:0.0:1761315622.606447:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315622.606830:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558016 00000100:00100000:3.0:1761315622.611081:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4598 00000100:00100000:3.0:1761315622.611096:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000359981a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307558016:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315622.611192:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000359981a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307558016:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 99us (4751us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315622.611201:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4598 00010000:00010000:1.0:1761315622.624214:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315623.131575:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322560512:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315623.131642:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315623.132053:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560512 00000100:00100000:2.0:1761315623.132108:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2874 00000100:00100000:2.0:1761315623.132114:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322560512:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315623.132146:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315623.132202:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322560512:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (560us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315623.132209:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2874 00000100:00100000:1.0:1761315623.132248:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322560512:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315623.643459:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315624.668232:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315625.692118:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315625.692278:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322560576:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315625.692364:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315625.692473:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322560640:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315625.692504:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315625.692543:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560640 00000100:00100000:3.0:1761315625.692584:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2875 00000100:00100000:3.0:1761315625.692590:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322560640:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315625.692623:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315625.692660:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322560640:12345-0@lo:41:osp-pre-0-1.0 Request processed in 70us (156us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315625.692667:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2875 00000100:00100000:3.0:1761315625.692750:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322560640:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315625.692860:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315625.692872:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322560768:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315625.692898:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315625.692903:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761315625.692907:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315625.692908:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315625.692914:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3868 00000100:00080000:2.0:1761315625.692917:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315625.692921:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315625.692927:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322560704:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.692943:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b59a8b32 x1846869322560960/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315625.692957:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315625.692964:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315625.692968:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315625.692970:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315625.692974:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3868 00000100:00080000:2.0:1761315625.692976:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315625.692979:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315625.692991:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560704 00000100:00100000:2.0:1761315625.692991:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bf1342a6 x1846869322561024/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315625.693023:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1702 00000100:00100000:1.0:1761315625.693029:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322560704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693047:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322560960:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693065:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322560704:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (109us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315625.693073:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1702 00000100:00100000:2.0:1761315625.693082:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315625.693101:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322561024:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693109:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322561152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693123:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315625.693144:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315625.693155:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560960 00000100:00100000:1.0:1761315625.693171:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f694cb8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322560704:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693175:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561024 00000100:00100000:1.0:1761315625.693187:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315625.693195:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322561088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693196:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2261 00000100:00100000:2.0:1761315625.693203:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322560960:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693220:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315625.693240:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322560832:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693257:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315625.693266:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322561216:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693278:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315625.693293:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:02020000:2.0:1761315625.693294:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315625.693299:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322560896:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693329:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315625.693366:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322560960:12345-0@lo:8:kworker/u8:2.0 Request processed in 140us (263us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315625.693375:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2261 00000100:00100000:1.0:1761315625.693379:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561216 00000100:00100000:2.0:1761315625.693393:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2262 00000100:00100000:2.0:1761315625.693397:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322561024:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693410:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1507 00000100:00100000:1.0:1761315625.693415:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322561216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315625.693442:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322561216:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315625.693448:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1507 00000100:00100000:1.0:1761315625.693469:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322561216:0@lo:400:kworker/u8:2.0 00010000:02020000:2.0:1761315625.693478:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315625.693506:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560768 00000100:00100000:2.0:1761315625.693509:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322561024:12345-0@lo:8:kworker/u8:2.0 Request processed in 110us (385us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315625.693517:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2262 00000100:00100000:1.0:1761315625.693530:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561152 00000100:00100000:2.0:1761315625.693542:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561088 00000100:00100000:1.0:1761315625.693546:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4599 00000100:00100000:1.0:1761315625.693551:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088e74441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322560768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693563:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560832 00000100:00100000:2.0:1761315625.693577:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4600 00000100:00100000:2.0:1761315625.693581:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e673437f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322561152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693625:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e673437f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322561152:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (484us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315625.693631:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4600 00000100:00100000:2.0:1761315625.693643:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4601 00000100:00100000:2.0:1761315625.693647:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3abc72e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322561088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315625.693663:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560576 00000100:00100000:2.0:1761315625.693676:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3abc72e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322561088:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (456us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315625.693683:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4601 00000100:00100000:0.0:1761315625.693695:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1001 00000100:00100000:2.0:1761315625.693699:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4602 00000100:00100000:0.0:1761315625.693711:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322560576:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315625.693717:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016b8e035 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322560832:12345-0@lo:400:kworker/u8:2.0 00002000:00000024:0.0:1761315625.693734:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315625.693754:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016b8e035 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322560832:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (496us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315625.693763:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4602 00000100:00100000:0.0:1761315625.693778:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322560576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (1413us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315625.693786:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1001 00000100:00080000:2.0:1761315625.693795:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315625.693800:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315625.693804:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59a8b32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322560960:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315625.693816:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315625.693818:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315625.693821:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322561024:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315625.693844:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322560832:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315625.693857:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322561088:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315625.693870:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322560576:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315625.693875:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322561152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315625.694276:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322560896 00000100:00100000:3.0:1761315625.694317:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2876 00000100:00100000:3.0:1761315625.694321:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322560896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315625.694395:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322560896:12345-0@lo:400:kworker/u8:2.0 Request processed in 72us (1065us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315625.694402:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2876 00000100:00100000:0.0:1761315625.694833:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322560896:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315625.696514:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088e74441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322560768:12345-0@lo:400:kworker/u8:2.0 Request processed in 2958us (3603us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315625.696528:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4599 00000100:00100000:2.0:1761315625.696818:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322560768:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315626.715866:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315627.727438:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315627.727791:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315627.730195:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315627.730325:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315627.730389:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558336 00000100:00100000:1.0:1761315627.730425:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2263 00000100:00100000:1.0:1761315627.730432:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b0c2887 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307558336:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315627.730554:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315627.730605:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b0c2887 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307558336:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 173us (281us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315627.730613:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2263 00000100:00100000:0.0:1761315627.730728:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558144 00000100:00100000:0.0:1761315627.730767:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1703 00000100:00100000:0.0:1761315627.730772:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315627.730804:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (3015us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315627.730810:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1703 00000100:00100000:3.0:1761315627.731614:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558208 00000100:00100000:3.0:1761315627.731660:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558272 00000100:00100000:3.0:1761315627.731680:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4603 00000100:00100000:3.0:1761315627.731685:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ac28ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307558208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315627.731742:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ac28ac7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307558208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (4312us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315627.731748:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4603 00000100:00100000:3.0:1761315627.731761:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4604 00000100:00100000:3.0:1761315627.731765:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000392f96aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315627.731786:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000392f96aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (1600us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315627.731791:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4604 00010000:00010000:0.0:1761315627.739505:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315628.251979:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315628.252520:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322561280:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315628.252586:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315628.254045:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561280 00000100:00100000:2.0:1761315628.254110:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2877 00000100:00100000:2.0:1761315628.254116:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322561280:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315628.254151:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315628.254209:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322561280:12345-0@lo:41:osp-pre-1-0.0 Request processed in 93us (1626us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315628.254215:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2877 00000100:00100000:1.0:1761315628.254360:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322561280:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315628.763667:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315629.790782:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315630.811498:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315630.811525:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315630.811546:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067bcaad5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322561344:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315630.811616:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315630.811644:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c3594b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322561472:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.811649:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322561408:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315630.811653:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315630.811659:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315630.811663:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315630.811677:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3873 00000100:00080000:3.0:1761315630.811680:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761315630.811764:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561344 00000100:00100000:0.0:1761315630.811778:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315630.811778:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.811801:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322561664:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315630.811815:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1002 00000100:00100000:2.0:1761315630.811822:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322561344:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315630.811829:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.811846:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315630.811850:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097cfe154 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322561600:0@lo:400:kworker/u8:2.0 00002000:00000024:2.0:1761315630.811863:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315630.811875:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.811918:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561664 00000100:00100000:2.0:1761315630.811918:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322561344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 94us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315630.811927:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1002 00000100:00100000:0.0:1761315630.811952:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2878 00000100:00100000:0.0:1761315630.811957:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322561664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315630.811966:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322561536:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812004:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322561664:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (175us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315630.812005:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.812015:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2878 00000100:00100000:2.0:1761315630.812025:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315630.812032:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561536 00000100:00100000:2.0:1761315630.812033:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322561728:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315630.812053:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315630.812057:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2879 00000100:00100000:0.0:1761315630.812061:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322561536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812077:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561408 00000100:00100000:3.0:1761315630.812078:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000be19034 x1846869322561792/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315630.812094:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322561536:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (94us total) trans 0 rc 0/0 00000100:00080000:3.0:1761315630.812096:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315630.812103:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315630.812106:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761315630.812107:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2879 00000100:00100000:1.0:1761315630.812107:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704 00000100:00080000:3.0:1761315630.812114:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3873 00000100:00100000:1.0:1761315630.812116:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322561408:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315630.812117:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315630.812122:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315630.812128:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561472 00000100:00100000:3.0:1761315630.812135:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d5431e2e x1846869322561856/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315630.812151:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322561408:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315630.812152:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561600 00000100:00100000:1.0:1761315630.812157:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704 00000100:00100000:0.0:1761315630.812167:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4605 00000100:00100000:0.0:1761315630.812170:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059ff679c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322561472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812199:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059ff679c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322561472:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (421us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315630.812199:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4606 00000100:00100000:1.0:1761315630.812203:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f7323fca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322561600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812205:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4605 00000100:00100000:1.0:1761315630.812231:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f7323fca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322561600:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (357us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315630.812236:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4606 00000100:00100000:0.0:1761315630.812240:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c3594b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322561472:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812267:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322561792:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812415:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097cfe154 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322561600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812425:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322561664:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812427:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315630.812432:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322561408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812444:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067bcaad5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322561344:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315630.812452:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561792 00000100:00100000:0.0:1761315630.812473:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2264 00000100:00100000:0.0:1761315630.812477:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322561792:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812551:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322561856:0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315630.812578:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315630.812581:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315630.812581:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322561984:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812608:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561856 00000100:00100000:0.0:1761315630.812617:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322561792:12345-0@lo:8:kworker/u8:2.0 Request processed in 139us (190us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315630.812626:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.812627:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2264 00000100:00100000:1.0:1761315630.812640:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2265 00000100:00100000:1.0:1761315630.812643:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322561856:12345-0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315630.812656:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315630.812660:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315630.812663:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322561792:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315630.812699:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322562048:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315630.812706:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315630.812730:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315630.812732:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322561536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315630.812748:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322561920:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.812769:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322561856:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (189us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315630.812776:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2265 00000100:00100000:1.0:1761315630.812797:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561728 00000100:00100000:1.0:1761315630.812815:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2880 00000100:00100000:1.0:1761315630.812818:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322561728:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315630.812836:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315630.812908:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322561728:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315630.812914:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2880 00000100:00080000:1.0:1761315630.812937:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315630.812940:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315630.812942:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322561856:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315630.813364:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561984 00000100:00100000:1.0:1761315630.813416:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315630.813524:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315630.813607:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322561728:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315630.813630:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322561920 00000100:00100000:0.0:1761315630.813637:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4607 00000100:00100000:0.0:1761315630.813644:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097cfe154 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322561984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315630.813660:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4608 00000100:00100000:2.0:1761315630.813675:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000883760dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322561920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315630.813694:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097cfe154 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322561984:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (1071us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315630.813701:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4607 00000100:00100000:2.0:1761315630.813718:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000883760dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322561920:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315630.813725:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4608 00000100:00100000:1.0:1761315630.813802:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322561920:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315630.813858:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562048 00000100:00100000:3.0:1761315630.813891:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1508 00000100:00100000:3.0:1761315630.813894:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322562048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315630.813940:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322562048:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (413us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315630.813946:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1508 00000100:00100000:3.0:1761315630.813973:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322561984:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315630.814040:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbfebe3b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322562048:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761315631.837820:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315632.841559:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315632.841672:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558592 00000100:00100000:2.0:1761315632.841720:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2266 00000100:00100000:2.0:1761315632.841727:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001458c5a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307558592:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315632.841819:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315632.841889:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001458c5a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307558592:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 161us (336us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315632.841897:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2266 00000100:00100000:0.0:1761315632.845436:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315632.847311:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558400 00000100:00100000:2.0:1761315632.847422:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1705 00000100:00100000:2.0:1761315632.847432:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315632.847521:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 89us (2090us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315632.847529:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1705 00000100:00100000:0.0:1761315632.847771:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315632.847889:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315632.848228:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558464 00000100:00100000:3.0:1761315632.848271:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558528 00000100:00100000:3.0:1761315632.848300:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4609 00000100:00100000:3.0:1761315632.848305:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004070f96b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307558464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315632.848376:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004070f96b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307558464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 70us (606us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315632.848384:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4609 00000100:00100000:3.0:1761315632.848398:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4610 00000100:00100000:3.0:1761315632.848401:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d643a28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558528:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315632.848427:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d643a28 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558528:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 24us (538us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315632.848433:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4610 00010000:00010000:0.0:1761315632.863591:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315633.374933:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322562112:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315633.375008:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315633.375136:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562112 00000100:00100000:1.0:1761315633.375187:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2881 00000100:00100000:1.0:1761315633.375193:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322562112:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315633.375224:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315633.375276:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322562112:12345-0@lo:41:osp-pre-1-0.0 Request processed in 81us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315633.375283:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2881 00000100:00100000:3.0:1761315633.375399:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000425b9659 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322562112:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315633.883525:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315634.907547:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315635.935877:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315635.935885:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315635.935888:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315635.935896:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3878 00000100:00080000:1.0:1761315635.935900:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315635.935906:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315635.935923:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000825eb04a x1846869322562496/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315635.935942:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315635.935945:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315635.935947:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315635.935950:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3878 00000100:00080000:1.0:1761315635.935951:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315635.935954:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315635.935964:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004f9b8f16 x1846869322562560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315635.936043:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322562176:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936108:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936135:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322562432:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936159:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936175:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322562816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936197:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936218:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315635.936224:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b358f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322562368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936246:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936261:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322562752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936280:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936313:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322562240:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936334:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936375:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322562624:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936397:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936412:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322562880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315635.936432:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936449:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315635.936454:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ab5a3502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322562304:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936472:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936485:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322562688:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936501:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936524:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000825eb04a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322562496:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936554:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936570:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322562560:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936589:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315635.936621:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562176 00000100:00100000:1.0:1761315635.936672:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1706 00000100:00100000:1.0:1761315635.936678:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322562176:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936724:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322562176:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (617us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.936731:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1706 00000100:00100000:1.0:1761315635.936762:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562432 00000100:00100000:1.0:1761315635.936782:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562304 00000100:00100000:1.0:1761315635.936796:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2882 00000100:00100000:1.0:1761315635.936799:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322562432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936825:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322562432:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (666us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.936830:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2882 00000100:00100000:1.0:1761315635.936839:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1761315635.936842:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322562304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936869:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322562304:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (397us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.936874:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2883 00000100:00100000:1.0:1761315635.936890:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562816 00000100:00100000:1.0:1761315635.936911:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562752 00000100:00100000:1.0:1761315635.936922:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562496 00000100:00100000:1.0:1761315635.936929:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562560 00000100:00100000:1.0:1761315635.936936:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1509 00000100:00100000:1.0:1761315635.936940:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322562816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.936966:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322562816:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (769us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.936972:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1509 00000100:00100000:1.0:1761315635.936982:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1510 00000100:00100000:1.0:1761315635.936985:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322562752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937007:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322562752:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (727us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937012:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1510 00000100:00100000:1.0:1761315635.937021:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2267 00000100:00100000:1.0:1761315635.937024:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322562496:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315635.937104:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315635.937134:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322562496:12345-0@lo:8:kworker/u8:2.0 Request processed in 109us (580us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315635.937140:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2267 00000100:00100000:1.0:1761315635.937149:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2268 00000100:00100000:1.0:1761315635.937152:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322562560:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315635.937225:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315635.937246:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322562560:12345-0@lo:8:kworker/u8:2.0 Request processed in 93us (657us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315635.937251:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2268 00000100:00100000:1.0:1761315635.937271:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562368 00000100:00100000:1.0:1761315635.937291:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562240 00000100:00100000:1.0:1761315635.937302:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562624 00000100:00100000:1.0:1761315635.937310:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562688 00000100:00100000:1.0:1761315635.937319:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4611 00000100:00100000:1.0:1761315635.937322:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322562368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937377:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322562368:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (1130us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937383:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4611 00000100:00100000:1.0:1761315635.937395:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4612 00000100:00100000:1.0:1761315635.937399:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322562240:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937422:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322562240:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (1088us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937427:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4612 00000100:00100000:1.0:1761315635.937436:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4613 00000100:00100000:1.0:1761315635.937439:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c45722b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322562624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937461:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c45722b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322562624:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (1064us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937467:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4613 00000100:00100000:1.0:1761315635.937475:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4614 00000100:00100000:1.0:1761315635.937478:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058b5fd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322562688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937502:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058b5fd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322562688:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1000us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937507:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4614 00000100:00100000:1.0:1761315635.937531:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562880 00000100:00100000:1.0:1761315635.937555:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1003 00000100:00100000:1.0:1761315635.937559:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322562880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315635.937590:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315635.937619:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322562880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (1186us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315635.937624:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1003 00000100:00080000:1.0:1761315635.937694:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315635.937698:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315635.937703:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000825eb04a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322562496:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315635.937712:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315635.937714:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315635.937716:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322562560:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937756:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ab5a3502 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322562304:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7274d4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322562688:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937774:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322562240:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937783:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077de439a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322562624:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937797:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a790d12c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322562880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315635.937842:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b358f44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322562368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937849:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322562752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937857:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322562176:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937866:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322562432:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315635.937875:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057deb555 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322562816:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761315635.938380:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315635.939212:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322562944:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315635.939265:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315635.940617:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322562944 00000100:00100000:0.0:1761315635.940683:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2884 00000100:00100000:0.0:1761315635.940688:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322562944:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315635.940724:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315635.945944:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322562944:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315635.946013:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322562944:12345-0@lo:41:osp-pre-0-1.0 Request processed in 5322us (6746us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315635.946022:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2884 00010000:00010000:0.0:1761315636.955679:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315637.960968:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315637.962025:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558656 00000100:00100000:2.0:1761315637.962093:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1707 00000100:00100000:2.0:1761315637.962100:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307558656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315637.962194:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307558656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 93us (1233us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315637.962201:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1707 00000100:00100000:3.0:1761315637.967505:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315637.967583:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558720 00000100:00100000:2.0:1761315637.967706:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4615 00000100:00100000:2.0:1761315637.967714:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307558720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315637.967793:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000425b9659 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307558720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 79us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315637.967801:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4615 00000100:00100000:3.0:1761315637.969674:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315637.971169:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315637.972027:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558848 00000100:00100000:0.0:1761315637.972061:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2269 00000100:00100000:0.0:1761315637.972066:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307558848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315637.972170:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315637.972244:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307558848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 176us (2576us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315637.972251:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2269 00000100:00100000:2.0:1761315637.973044:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558784 00000100:00100000:2.0:1761315637.973075:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4616 00000100:00100000:2.0:1761315637.973079:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315637.973133:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbfebe3b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307558784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 52us (1968us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315637.973148:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4616 00010000:00010000:1.0:1761315637.979783:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315638.491592:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315638.491611:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322563008:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315638.491721:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315638.491807:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563008 00000100:00100000:3.0:1761315638.491866:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2885 00000100:00100000:3.0:1761315638.491872:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322563008:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315638.491910:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315638.491957:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322563008:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315638.491964:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2885 00000100:00100000:2.0:1761315638.492057:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322563008:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315639.003553:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315640.027495:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315641.051622:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315641.051753:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315641.051759:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315641.051763:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315641.051775:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3883 00000100:00080000:3.0:1761315641.051778:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315641.051783:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315641.051804:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e3d6aa99 x1846869322563392/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315641.051822:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315641.051825:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315641.051827:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315641.051830:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3883 00000100:00080000:3.0:1761315641.051831:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315641.051834:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315641.051844:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e25742f6 x1846869322563456/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315641.051933:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322563264:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.051993:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.052030:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322563648:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052053:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.052075:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1761315641.052080:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322563072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052104:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.052116:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322563328:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052135:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.052150:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322563712:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052168:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.052198:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563648 00000100:00100000:0.0:1761315641.052229:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563712 00000100:00100000:0.0:1761315641.052244:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1511 00000100:00100000:0.0:1761315641.052249:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322563648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052300:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322563648:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (247us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315641.052307:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1511 00000100:00100000:0.0:1761315641.052321:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1512 00000100:00100000:0.0:1761315641.052325:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322563712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052370:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322563712:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (201us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315641.052376:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1512 00000100:00100000:0.0:1761315641.052403:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322563712:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052412:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322563648:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052436:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563328 00000100:00100000:0.0:1761315641.052457:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2886 00000100:00100000:0.0:1761315641.052460:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322563328:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.052490:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322563328:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (354us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315641.052495:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2886 00000100:00100000:0.0:1761315641.052515:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322563328:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052591:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322563136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052629:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052656:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322563520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052676:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052691:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315641.052696:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322563200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052712:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052725:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322563584:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052743:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052765:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322563392:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052786:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052798:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322563456:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315641.052815:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315641.052839:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563392 00000100:00100000:1.0:1761315641.052857:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563456 00000100:00100000:1.0:1761315641.052866:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2270 00000100:00100000:1.0:1761315641.052870:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322563392:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315641.052966:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315641.052996:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322563392:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (210us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315641.053002:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2270 00000100:00100000:1.0:1761315641.053012:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2271 00000100:00100000:1.0:1761315641.053015:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322563456:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315641.053092:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315641.053113:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322563456:12345-0@lo:8:kworker/u8:2.0 Request processed in 97us (298us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315641.053118:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2271 00000100:00100000:1.0:1761315641.053135:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563072 00000100:00100000:1.0:1761315641.053163:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1708 00000100:00100000:1.0:1761315641.053167:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322563072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053193:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322563072:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1089us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053199:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1708 00000100:00100000:1.0:1761315641.053230:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563200 00000100:00100000:1.0:1761315641.053249:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2887 00000100:00100000:1.0:1761315641.053252:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322563200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053282:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322563200:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (569us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053288:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2887 00000100:00100000:1.0:1761315641.053305:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563264 00000100:00100000:1.0:1761315641.053323:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563136 00000100:00100000:1.0:1761315641.053337:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563520 00000100:00100000:1.0:1761315641.053369:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563584 00000100:00100000:1.0:1761315641.053380:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4617 00000100:00100000:1.0:1761315641.053384:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e59c3bf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322563264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053408:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e59c3bf1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322563264:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (1416us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053413:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4617 00000100:00100000:1.0:1761315641.053423:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4618 00000100:00100000:1.0:1761315641.053426:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322563136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053449:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322563136:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (821us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053454:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4618 00000100:00100000:1.0:1761315641.053463:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4619 00000100:00100000:1.0:1761315641.053466:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab5a1368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322563520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053487:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab5a1368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322563520:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (811us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053492:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4619 00000100:00100000:1.0:1761315641.053500:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4620 00000100:00100000:1.0:1761315641.053503:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfd01862 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322563584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053523:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfd01862 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322563584:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (780us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315641.053528:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4620 00000100:00080000:1.0:1761315641.053573:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315641.053577:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315641.053580:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322563392:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315641.053588:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315641.053591:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315641.053593:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322563456:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053635:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322563200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053645:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322563584:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053653:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322563136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315641.053660:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322563520:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.053762:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322563072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315641.053773:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322563264:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315641.053958:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315641.053967:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322563776:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315641.053996:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315641.054135:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563776 00000100:00100000:3.0:1761315641.054171:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1004 00000100:00100000:3.0:1761315641.054175:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322563776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315641.054196:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315641.054227:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322563776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (231us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315641.054233:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1004 00000100:00100000:1.0:1761315641.054316:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315641.054324:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008824de23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322563840:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315641.054369:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315641.054438:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563840 00000100:00100000:0.0:1761315641.054470:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2888 00000100:00100000:0.0:1761315641.054474:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052a02902 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322563840:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315641.054496:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315641.054534:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052a02902 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322563840:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (165us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315641.054540:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2888 00000100:00100000:3.0:1761315641.054704:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008824de23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322563840:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315641.054810:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322563776:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1761315642.075758:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315643.083698:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315643.084221:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315643.084259:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315643.084284:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315643.084322:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559040 00000100:00100000:0.0:1761315643.084387:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558976 00000100:00100000:0.0:1761315643.084408:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4621 00000100:00100000:0.0:1761315643.084415:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067acb148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315643.084489:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067acb148 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559040:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 74us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315643.084497:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4621 00000100:00100000:0.0:1761315643.084513:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4622 00000100:00100000:0.0:1761315643.084516:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000087b042 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307558976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315643.084538:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000087b042 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307558976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315643.084544:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4622 00000100:00100000:1.0:1761315643.084734:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559104 00000100:00100000:1.0:1761315643.084766:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2272 00000100:00100000:1.0:1761315643.084771:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f0aeaf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307559104:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315643.084860:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315643.084889:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f0aeaf6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307559104:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 117us (1198us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315643.084895:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2272 00000100:00100000:2.0:1761315643.085394:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307558912 00000100:00100000:2.0:1761315643.085432:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1709 00000100:00100000:2.0:1761315643.085436:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315643.085471:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307558912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (1251us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315643.085478:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1709 00010000:00010000:0.0:1761315643.099492:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315643.613212:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315643.617589:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322563904:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315643.617682:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315643.617899:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563904 00000100:00100000:3.0:1761315643.618633:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2889 00000100:00100000:3.0:1761315643.618640:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c91750f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322563904:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315643.618674:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315643.618734:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c91750f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322563904:12345-0@lo:41:osp-pre-1-0.0 Request processed in 93us (1057us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315643.618741:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2889 00000100:00100000:0.0:1761315643.618767:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322563904:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315644.123544:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315645.148093:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315646.172056:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315646.172063:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315646.172067:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315646.172075:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3888 00000100:00080000:2.0:1761315646.172077:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315646.172082:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315646.172101:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004564b6ec x1846869322564288/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315646.172123:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315646.172126:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315646.172128:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315646.172131:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3888 00000100:00080000:2.0:1761315646.172133:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315646.172136:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315646.172146:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5d13e x1846869322564352/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315646.172199:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564096:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172261:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315646.172287:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322564480:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172312:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315646.172329:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315646.172334:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057a1f78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322564032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172377:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315646.172393:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078f5cb75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322564416:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172412:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315646.172443:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564480 00000100:00100000:2.0:1761315646.172483:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564032 00000100:00100000:2.0:1761315646.172497:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564416 00000100:00100000:2.0:1761315646.172512:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4623 00000100:00100000:2.0:1761315646.172516:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322564480:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172599:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322564480:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315646.172606:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4623 00000100:00100000:2.0:1761315646.172622:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4624 00000100:00100000:2.0:1761315646.172625:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322564032:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172652:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ddb6462d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322564032:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315646.172657:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4624 00000100:00100000:2.0:1761315646.172667:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4625 00000100:00100000:2.0:1761315646.172671:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322564416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172694:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006047b3cd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322564416:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (282us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315646.172699:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4625 00000100:00100000:2.0:1761315646.172738:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057a1f78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322564032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172746:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078f5cb75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322564416:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.172754:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000373dca84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322564480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173203:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322563968:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173243:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315646.173272:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564224:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173294:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315646.173309:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564608:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173330:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315646.173368:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315646.173373:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000482873b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322564160:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173395:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315646.173409:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a460ec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322564544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173427:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315646.173456:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564096 00000100:00100000:3.0:1761315646.173480:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564224 00000100:00100000:3.0:1761315646.173495:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2890 00000100:00100000:3.0:1761315646.173499:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067bcaad5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564096:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173534:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067bcaad5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564096:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (1274us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315646.173539:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2890 00000100:00100000:3.0:1761315646.173551:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2891 00000100:00100000:3.0:1761315646.173554:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173594:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564224:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315646.173599:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2891 00000100:00100000:3.0:1761315646.173614:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564608 00000100:00100000:3.0:1761315646.173630:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564544 00000100:00100000:3.0:1761315646.173640:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1513 00000100:00100000:3.0:1761315646.173643:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173667:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564608:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315646.173671:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1513 00000100:00100000:3.0:1761315646.173679:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1514 00000100:00100000:3.0:1761315646.173682:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322564544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173704:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322564544:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (278us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315646.173708:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1514 00000100:00100000:3.0:1761315646.173724:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564160 00000100:00100000:3.0:1761315646.173746:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4626 00000100:00100000:3.0:1761315646.173749:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322564160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173773:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000535a6e35 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322564160:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315646.173778:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4626 00000100:00100000:3.0:1761315646.173826:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000482873b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322564160:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173834:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a460ec1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322564544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173842:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564224:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173849:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564608:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315646.173880:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564096:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315646.174013:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004564b6ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322564288:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315646.174043:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315646.174062:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322564352:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315646.174082:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315646.174125:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564288 00000100:00100000:1.0:1761315646.174144:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564352 00000100:00100000:1.0:1761315646.174154:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2273 00000100:00100000:1.0:1761315646.174157:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322564288:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315646.174253:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315646.174285:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322564288:12345-0@lo:8:kworker/u8:0.0 Request processed in 127us (241us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315646.174291:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2273 00000100:00100000:1.0:1761315646.174302:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2274 00000100:00100000:1.0:1761315646.174305:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322564352:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315646.174401:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315646.174426:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322564352:12345-0@lo:8:kworker/u8:0.0 Request processed in 119us (343us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315646.174432:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2274 00000100:00080000:1.0:1761315646.174476:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315646.174481:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315646.174484:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004564b6ec pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322564288:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315646.174494:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315646.174496:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315646.174499:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322564352:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315646.174527:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315646.174534:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564672:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315646.174575:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315646.175993:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564672 00000100:00100000:0.0:1761315646.176036:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1005 00000100:00100000:0.0:1761315646.176041:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322564672:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315646.176082:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315646.176146:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322564672:12345-0@lo:13:osp-pre-1-1.0 Request processed in 104us (1587us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315646.176153:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1005 00010000:00010000:0.0:1761315646.176217:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315646.176230:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002deaa2ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564736:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315646.176259:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315646.176298:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564672:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315646.176326:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564736 00000100:00100000:0.0:1761315646.176368:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2892 00000100:00100000:0.0:1761315646.176371:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000c58668 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564736:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315646.176393:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315646.176425:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000c58668 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564736:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (166us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315646.176430:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2892 00000100:00100000:0.0:1761315646.176456:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002deaa2ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564736:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315646.177789:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322563968 00000100:00100000:2.0:1761315646.177846:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1710 00000100:00100000:2.0:1761315646.177852:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322563968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315646.177917:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322563968:12345-0@lo:400:kworker/u8:0.0 Request processed in 65us (4675us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315646.177924:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1710 00000100:00100000:3.0:1761315646.177996:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322563968:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315647.196220:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315648.202465:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315648.205080:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559232 00000100:00100000:3.0:1761315648.205292:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315648.205523:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315648.205585:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559168 00000100:00100000:0.0:1761315648.205615:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1711 00000100:00100000:0.0:1761315648.205620:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315648.205681:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 60us (392us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315648.205687:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1711 00000100:00100000:1.0:1761315648.208298:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559296 00000100:00100000:1.0:1761315648.208356:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4627 00000100:00100000:1.0:1761315648.208363:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7b15fb6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307559232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315648.208463:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4628 00000100:00100000:2.0:1761315648.208469:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9137a0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559296:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315648.209658:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315648.209750:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7b15fb6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307559232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1386us (7290us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315648.209760:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4627 00000100:00100000:2.0:1761315648.210084:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9137a0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559296:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1613us (4560us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315648.210093:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4628 00000100:00100000:0.0:1761315648.211285:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559360 00000100:00100000:2.0:1761315648.213701:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2275 00000100:00100000:2.0:1761315648.213713:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307559360:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315648.213806:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315648.214760:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307559360:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 1044us (5102us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315648.214770:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2275 00010000:00010000:3.0:1761315648.224870:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315648.731650:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315648.732179:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564800:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315648.732249:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315648.732318:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564800 00000100:00100000:2.0:1761315648.732403:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2893 00000100:00100000:2.0:1761315648.732409:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002deaa2ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564800:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315648.732442:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315648.732487:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002deaa2ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322564800:12345-0@lo:41:osp-pre-1-0.0 Request processed in 77us (238us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315648.732493:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2893 00000100:00100000:2.0:1761315648.732535:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322564800:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315649.243549:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315650.267860:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315651.291656:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315651.291663:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315651.291667:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315651.291676:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3893 00000100:00080000:3.0:1761315651.291679:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315651.291685:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315651.291704:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e25742f6 x1846869322565184/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315651.291726:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315651.291730:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315651.291731:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315651.291734:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3893 00000100:00080000:3.0:1761315651.291736:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315651.291738:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315651.291749:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e3d6aa99 x1846869322565248/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315651.291834:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322564864:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.291899:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.291933:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322565120:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.291957:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.291976:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b623220c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322565504:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.291999:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.292021:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761315651.292026:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322565056:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292049:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.292064:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322565440:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292085:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.292120:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565120 00000100:00100000:2.0:1761315651.292160:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2894 00000100:00100000:2.0:1761315651.292166:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322565120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292212:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322565120:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (255us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.292219:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2894 00000100:00100000:2.0:1761315651.292244:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565504 00000100:00100000:2.0:1761315651.292264:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565440 00000100:00100000:2.0:1761315651.292278:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1515 00000100:00100000:2.0:1761315651.292281:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322565504:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292309:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322565504:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.292314:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1515 00000100:00100000:2.0:1761315651.292325:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1516 00000100:00100000:2.0:1761315651.292328:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c45a5c1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322565440:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292372:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c45a5c1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322565440:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (287us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.292377:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1516 00000100:00100000:2.0:1761315651.292395:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565056 00000100:00100000:2.0:1761315651.292415:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4629 00000100:00100000:2.0:1761315651.292419:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004564b6ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322565056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292445:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004564b6ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322565056:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (396us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.292450:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4629 00000100:00100000:2.0:1761315651.292487:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322565056:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292496:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322565440:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292504:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322565120:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.292517:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b623220c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322565504:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315651.292972:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564864 00000100:00100000:3.0:1761315651.293018:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1712 00000100:00100000:3.0:1761315651.293022:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322564864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.293039:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564992:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315651.293069:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322564864:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (1170us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315651.293076:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1712 00000100:00100000:0.0:1761315651.293083:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315651.293105:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322565376:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.293131:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315651.293149:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322564928:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315651.293179:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315651.293196:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322565312:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315651.293218:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315651.293235:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565568:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315651.293239:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565376 00000100:00100000:3.0:1761315651.293257:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315651.293272:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564928 00010000:00010000:3.0:1761315651.293287:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315651.293299:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565312 00000100:00100000:3.0:1761315651.293304:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4630 00000100:00100000:0.0:1761315651.293320:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4631 00000100:00100000:0.0:1761315651.293325:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322564928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.293374:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f576ac9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322564928:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315651.293380:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4631 00000100:00100000:0.0:1761315651.293391:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4632 00000100:00100000:0.0:1761315651.293394:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322565312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.293416:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5ec4a5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322565312:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (199us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315651.293421:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4632 00000100:00100000:0.0:1761315651.293449:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322564928:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.293457:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322565312:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315651.293695:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315651.293706:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565632:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315651.293736:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315651.293763:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565632 00000100:00100000:1.0:1761315651.293788:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1006 00000100:00100000:1.0:1761315651.293792:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322565632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315651.293812:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315651.293843:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322565632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (107us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315651.293849:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1006 00000100:00100000:1.0:1761315651.293877:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565632:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315651.294108:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322565184:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315651.294135:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.294159:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322565248:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315651.294177:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315651.294235:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322564992 00000100:00100000:2.0:1761315651.294256:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2895 00000100:00100000:2.0:1761315651.294260:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2541175 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.294293:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2541175 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322564992:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (1213us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.294299:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2895 00000100:00100000:2.0:1761315651.294310:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565568 00000100:00100000:2.0:1761315651.294324:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2896 00000100:00100000:2.0:1761315651.294327:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322565568:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315651.294363:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315651.294391:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322565568:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (1135us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315651.294398:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2896 00000100:00100000:2.0:1761315651.294419:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322564992:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315651.294447:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322564864:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.294538:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565184 00000100:00100000:0.0:1761315651.294562:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565248 00000100:00100000:0.0:1761315651.294578:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2276 00000100:00100000:0.0:1761315651.294581:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000354757ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322565184:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315651.294674:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315651.294706:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000354757ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322565184:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (571us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315651.294712:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2276 00000100:00100000:0.0:1761315651.294723:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2277 00000100:00100000:0.0:1761315651.294726:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322565248:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315651.294801:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315651.294820:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b372c039 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322565376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315651.294825:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322565248:12345-0@lo:8:kworker/u8:2.0 Request processed in 97us (647us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315651.294831:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2277 00000100:00080000:0.0:1761315651.294858:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315651.294863:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315651.294867:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322565184:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315651.294871:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b372c039 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322565376:12345-0@lo:400:kworker/u8:2.0 Request processed in 1557us (1736us total) trans 0 rc 0/0 00000100:00080000:0.0:1761315651.294880:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315651.294880:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4630 00000100:00080000:0.0:1761315651.294886:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315651.294890:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322565248:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315651.294920:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322565376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315651.294986:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565568:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315652.315604:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315653.325599:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315653.327582:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315653.327618:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315653.328013:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559424 00000100:00100000:1.0:1761315653.328055:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1713 00000100:00100000:1.0:1761315653.328061:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315653.328130:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559424:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 68us (2540us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315653.328136:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1713 00000100:00100000:3.0:1761315653.328900:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315653.328956:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559616 00000100:00100000:2.0:1761315653.328988:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559488 00000100:00100000:1.0:1761315653.329000:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2278 00000100:00100000:1.0:1761315653.329006:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307559616:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761315653.329020:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559552 00000100:00100000:2.0:1761315653.329037:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4633 00000100:00100000:2.0:1761315653.329042:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307559488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315653.329080:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf2823d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307559488:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 37us (1504us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315653.329086:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4633 00000100:00100000:2.0:1761315653.329098:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4634 00000100:00100000:2.0:1761315653.329103:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559552:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00010000:02020000:1.0:1761315653.329105:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315653.329124:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065a64462 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559552:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (1506us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315653.329130:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4634 00000100:00100000:1.0:1761315653.329150:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307559616:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 142us (251us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315653.329157:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2278 00010000:00010000:2.0:1761315653.341619:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315653.852084:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322565696:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315653.852153:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315653.853219:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565696 00000100:00100000:2.0:1761315653.853275:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2897 00000100:00100000:2.0:1761315653.853281:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322565696:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315653.853313:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315653.853986:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322565696:12345-0@lo:41:osp-pre-1-0.0 Request processed in 702us (1831us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315653.853994:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2897 00000100:00100000:1.0:1761315653.854070:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322565696:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315654.364121:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315655.387521:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315656.411609:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315656.411789:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322565760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315656.411862:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.412023:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565760 00000100:00100000:1.0:1761315656.412084:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1007 00000100:00100000:1.0:1761315656.412091:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322565760:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315656.412123:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315656.412268:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322565760:12345-0@lo:13:osp-pre-1-1.0 Request processed in 176us (406us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.412276:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1007 00000100:00100000:2.0:1761315656.412456:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565824:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315656.412503:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315656.412578:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322565760:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315656.412624:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565824 00000100:00100000:0.0:1761315656.412651:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2898 00000100:00100000:0.0:1761315656.412654:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322565824:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315656.412676:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315656.413045:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322565824:12345-0@lo:41:osp-pre-0-1.0 Request processed in 388us (540us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315656.413052:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2898 00000100:00100000:0.0:1761315656.413203:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322565952:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315656.413237:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315656.413829:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315656.413834:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315656.413837:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315656.413842:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3898 00000100:00080000:1.0:1761315656.413844:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315656.413848:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315656.413866:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000275a380d x1846869322566144/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315656.413882:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315656.413885:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315656.413887:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315656.413890:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3898 00000100:00080000:1.0:1761315656.413892:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315656.413894:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315656.413905:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009f990ca7 x1846869322566208/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315656.413965:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565952 00000100:00100000:1.0:1761315656.413999:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4635 00000100:00100000:1.0:1761315656.414005:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322565952:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414047:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008824de23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322565952:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (810us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.414054:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4635 00000100:00100000:1.0:1761315656.414073:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322566144:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414099:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.414114:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322566208:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414150:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.414176:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566144 00000100:00100000:1.0:1761315656.414193:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566208 00000100:00100000:1.0:1761315656.414204:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2279 00000100:00100000:1.0:1761315656.414207:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322566144:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315656.414292:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315656.414325:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322566144:12345-0@lo:8:kworker/u8:2.0 Request processed in 116us (225us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315656.414332:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2279 00000100:00100000:1.0:1761315656.414363:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2280 00000100:00100000:1.0:1761315656.414367:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322566208:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315656.414439:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315656.414463:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322566208:12345-0@lo:8:kworker/u8:2.0 Request processed in 95us (314us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315656.414468:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2280 00000100:00080000:1.0:1761315656.414506:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315656.414509:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315656.414512:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322566144:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315656.414520:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315656.414522:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315656.414524:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f990ca7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322566208:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414547:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322566016:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414572:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.414585:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322566400:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414602:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.414613:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315656.414617:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322566336:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414636:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315656.414659:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566016 00000100:00100000:1.0:1761315656.414680:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566336 00000100:00100000:1.0:1761315656.414697:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4636 00000100:00100000:1.0:1761315656.414700:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322566016:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414732:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322566016:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.414738:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4636 00000100:00100000:1.0:1761315656.414748:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4637 00000100:00100000:1.0:1761315656.414751:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322566336:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414775:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322566336:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (138us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.414780:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4637 00000100:00100000:1.0:1761315656.414794:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566400 00000100:00100000:1.0:1761315656.414812:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1517 00000100:00100000:1.0:1761315656.414815:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322566400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414839:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322566400:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (237us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.414844:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1517 00000100:00100000:1.0:1761315656.414870:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322566336:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414878:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322566016:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.414884:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322566400:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415121:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a4fe2a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322565824:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315656.415141:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322566080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415170:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315656.415205:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315656.415211:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322565888:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415239:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315656.415256:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322566272:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415276:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315656.415300:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566272 00000100:00100000:2.0:1761315656.415326:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4638 00000100:00100000:2.0:1761315656.415330:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322566272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415389:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000373dca84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322566272:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (112us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315656.415395:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4638 00000100:00100000:2.0:1761315656.415417:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322565888 00000100:00100000:2.0:1761315656.415439:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1714 00000100:00100000:2.0:1761315656.415443:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322565888:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415471:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322565888:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315656.415477:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1714 00000100:00100000:2.0:1761315656.415523:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322565888:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315656.415533:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322566272:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315656.415630:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322565952:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.416807:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566080 00000100:00100000:1.0:1761315656.417654:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2899 00000100:00100000:1.0:1761315656.417661:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000482873b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322566080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315656.418031:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000482873b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322566080:12345-0@lo:400:kworker/u8:2.0 Request processed in 369us (2860us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315656.418038:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2899 00000100:00100000:2.0:1761315656.418095:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322566080:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315657.435671:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315658.440973:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315658.441136:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559680 00000100:00100000:2.0:1761315658.441193:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1715 00000100:00100000:2.0:1761315658.441200:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307559680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315658.441276:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307559680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (310us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315658.441285:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1715 00000100:00100000:2.0:1761315658.441817:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315658.441851:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315658.442077:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559744 00000100:00100000:0.0:1761315658.442112:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559808 00000100:00100000:0.0:1761315658.442130:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4639 00000100:00100000:0.0:1761315658.442134:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307559744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315658.442177:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078f5cb75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307559744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 43us (363us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315658.442184:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4639 00000100:00100000:0.0:1761315658.442196:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4640 00000100:00100000:0.0:1761315658.442200:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057a1f78f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315658.442222:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057a1f78f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307559808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (372us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315658.442228:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4640 00000100:00100000:2.0:1761315658.442335:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315658.442428:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559872 00000100:00100000:1.0:1761315658.442461:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2281 00000100:00100000:1.0:1761315658.442466:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307559872:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315658.442562:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315658.442590:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307559872:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 124us (256us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315658.442595:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2281 00010000:00010000:2.0:1761315658.459913:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315658.972766:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315658.972786:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322566464:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315658.972846:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315658.972930:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566464 00000100:00100000:3.0:1761315658.972977:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2900 00000100:00100000:3.0:1761315658.972982:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322566464:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315658.973011:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315658.973051:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322566464:12345-0@lo:41:osp-pre-1-0.0 Request processed in 68us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315658.973057:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2900 00000100:00100000:1.0:1761315658.973355:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000acbc995e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322566464:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315659.483853:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315660.516578:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315661.531791:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315661.531806:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322566528:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315661.531868:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761315661.531911:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315661.532015:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566528 00000100:00100000:3.0:1761315661.532143:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2901 00000100:00100000:3.0:1761315661.532149:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322566528:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315661.532182:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315661.532232:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322566528:12345-0@lo:41:osp-pre-0-1.0 Request processed in 82us (364us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315661.532240:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2901 00000100:00100000:2.0:1761315661.532328:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011fc2a84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322566528:0@lo:41:osp-pre-0-1.0 00000100:00080000:2.0:1761315661.536840:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315661.536847:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315661.536851:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315661.536860:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3903 00000100:00080000:2.0:1761315661.536864:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315661.536869:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315661.536888:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000035a2ab8d x1846869322566912/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315661.536900:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315661.536902:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315661.536904:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315661.536907:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3903 00000100:00080000:2.0:1761315661.536909:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315661.536911:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315661.536918:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000604dcad7 x1846869322566976/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315661.536998:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322566912:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537055:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315661.537086:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322566976:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537102:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315661.537142:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566912 00000100:00100000:1.0:1761315661.537172:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566976 00000100:00100000:0.0:1761315661.537175:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322566784:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537189:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2282 00000100:00100000:1.0:1761315661.537196:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322566912:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537223:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315661.537241:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322567168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537264:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315661.537281:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315661.537285:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322566720:0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315661.537289:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315661.537309:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315661.537327:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322567104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537338:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322566912:12345-0@lo:8:kworker/u8:0.0 Request processed in 141us (282us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315661.537364:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2282 00000100:00100000:0.0:1761315661.537377:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315661.537380:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567168 00000100:00100000:1.0:1761315661.537412:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1518 00000100:00100000:0.0:1761315661.537416:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566784 00000100:00100000:1.0:1761315661.537416:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322567168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537449:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566720 00000100:00100000:1.0:1761315661.537473:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322567168:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (207us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315661.537483:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567104 00000100:00100000:1.0:1761315661.537483:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1518 00000100:00100000:1.0:1761315661.537499:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2283 00000100:00100000:1.0:1761315661.537504:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322566976:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537508:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4641 00000100:00100000:0.0:1761315661.537514:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322566784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537545:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b04bb871 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322566784:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315661.537550:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4641 00000100:00100000:0.0:1761315661.537561:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4642 00000100:00100000:0.0:1761315661.537565:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322566720:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315661.537583:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315661.537591:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034d7daff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322566720:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (282us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315661.537597:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4642 00000100:00100000:0.0:1761315661.537610:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4643 00000100:00100000:1.0:1761315661.537613:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322566976:12345-0@lo:8:kworker/u8:0.0 Request processed in 109us (509us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315661.537618:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322567104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537624:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2283 00000100:00100000:0.0:1761315661.537654:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3fb549f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322567104:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (278us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315661.537660:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4643 00000100:00080000:1.0:1761315661.537694:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315661.537698:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315661.537702:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322566912:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537706:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e25742f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322566720:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315661.537714:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315661.537718:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315661.537721:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604dcad7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322566976:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537726:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e34014aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322567104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537742:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322566784:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537751:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c61786a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322567168:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537761:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322566592:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315661.537789:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315661.537811:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566592 00000100:00100000:1.0:1761315661.537837:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322566848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.537841:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1008 00000100:00100000:0.0:1761315661.537847:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322566592:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315661.537860:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322566656:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.537865:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:0.0:1761315661.537867:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315661.537885:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322567232:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315661.537913:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315661.537973:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566656 00000100:00100000:2.0:1761315661.538001:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1716 00000100:00100000:2.0:1761315661.538004:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322566656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.539783:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322566592:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1929us (1989us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315661.539798:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1008 00000100:00100000:3.0:1761315661.539837:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322567040:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.539847:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322566848 00000100:00100000:3.0:1761315661.539893:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315661.539931:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315661.539969:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567040 00000100:00100000:1.0:1761315661.539994:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004f9b8f16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322566592:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315661.540005:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4644 00000100:00100000:3.0:1761315661.540012:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322567040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.540032:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567232 00000100:00100000:1.0:1761315661.540064:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1519 00000100:00100000:3.0:1761315661.540067:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011fc2a84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322567040:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315661.540072:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322567232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315661.540081:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4644 00000100:00100000:3.0:1761315661.540114:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322567040:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315661.540194:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322567232:12345-0@lo:400:kworker/u8:0.0 Request processed in 123us (264us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315661.540201:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1519 00000100:00100000:3.0:1761315661.540225:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322567232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315661.540286:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322566656:12345-0@lo:400:kworker/u8:0.0 Request processed in 2279us (2373us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315661.540294:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1716 00000100:00100000:3.0:1761315661.540386:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3d6aa99 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322566656:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.540636:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2902 00000100:00100000:0.0:1761315661.540641:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322566848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.540691:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322566848:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (2829us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315661.540697:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003f0b33c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322566848:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315661.540701:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2902 00010000:00010000:2.0:1761315662.557813:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315663.560508:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315663.560556:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315663.560672:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307559936 00000100:00100000:2.0:1761315663.562114:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1717 00000100:00100000:2.0:1761315663.562123:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315663.562191:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307559936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 69us (1693us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315663.562199:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1717 00000100:00100000:3.0:1761315663.563250:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315663.564390:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315663.565085:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560128 00000100:00100000:1.0:1761315663.565132:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2284 00000100:00100000:1.0:1761315663.565139:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307560128:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315663.565232:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315663.565292:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307560128:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 153us (906us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315663.565299:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2284 00000100:00100000:0.0:1761315663.565906:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560000 00000100:00100000:0.0:1761315663.565949:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560064 00000100:00100000:0.0:1761315663.565969:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4645 00000100:00100000:0.0:1761315663.565974:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000acbc995e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307560000:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315663.566022:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000acbc995e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307560000:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 47us (5466us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315663.566027:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4646 00000100:00100000:0.0:1761315663.566037:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4645 00000100:00100000:2.0:1761315663.566038:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a4fe2a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307560064:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315663.566075:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a4fe2a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307560064:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 40us (2828us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315663.566083:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4646 00010000:00010000:3.0:1761315663.579800:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315664.092031:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322567296:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315664.092105:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315664.092163:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567296 00000100:00100000:3.0:1761315664.093753:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2903 00000100:00100000:3.0:1761315664.093764:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322567296:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315664.093807:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315664.093885:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322567296:12345-0@lo:41:osp-pre-1-0.0 Request processed in 123us (1781us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315664.093892:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2903 00000100:00100000:0.0:1761315664.094016:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322567296:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315664.604308:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315665.627642:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315666.651997:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1761315666.652001:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315666.652005:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315666.652009:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315666.652018:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3909 00000100:00080000:2.0:1761315666.652022:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315666.652028:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315666.652049:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d48b7a5 x1846869322567680/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315666.652075:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315666.652078:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315666.652080:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315666.652084:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3909 00000100:00080000:2.0:1761315666.652087:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315666.652091:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315666.652100:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322567680:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652105:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a3d67ed x1846869322567744/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315666.652169:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315666.652208:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322567744:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652222:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567680 00000100:00100000:3.0:1761315666.652230:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315666.652255:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567744 00000100:00100000:2.0:1761315666.652287:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2285 00000100:00100000:2.0:1761315666.652293:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322567680:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652429:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2286 00000100:00100000:3.0:1761315666.652433:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322567744:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315666.652437:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315666.652482:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322567680:12345-0@lo:8:kworker/u8:2.0 Request processed in 188us (317us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315666.652490:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2285 00000100:00100000:0.0:1761315666.652528:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322567424:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761315666.652528:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315666.652565:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322567744:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (334us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315666.652572:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315666.652575:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2286 00000100:00100000:0.0:1761315666.652594:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322567808:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315666.652606:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315666.652609:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315666.652614:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322567552:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652615:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322567744:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315666.652619:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315666.652630:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315666.652632:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315666.652634:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322568064:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315666.652635:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322567680:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652653:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315666.652664:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315666.652676:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322567936:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652678:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567424 00000100:00100000:0.0:1761315666.652683:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:0.0:1761315666.652690:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322567360:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652706:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315666.652708:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567808 00000100:00100000:0.0:1761315666.652721:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315666.652728:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567552 00000100:00100000:3.0:1761315666.652731:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315666.652740:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322567488:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315666.652741:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322567616:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652749:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4647 00000100:00100000:2.0:1761315666.652754:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322567424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652780:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315666.652789:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567360 00000100:00100000:3.0:1761315666.652808:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322567872:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315666.652809:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567936 00000100:00100000:2.0:1761315666.652809:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000137dd527 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322567424:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (237us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315666.652819:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1718 00000100:00100000:2.0:1761315666.652819:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4647 00000100:00100000:0.0:1761315666.652823:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322567360:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652836:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4648 00000100:00100000:2.0:1761315666.652840:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322567808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652847:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315666.652855:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1520 00000100:00100000:0.0:1761315666.652866:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322567360:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (144us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315666.652872:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322567936:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.652874:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000889dbf19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322567808:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315666.652875:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1718 00000100:00100000:3.0:1761315666.652883:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568064 00000100:00100000:2.0:1761315666.652884:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4648 00000100:00100000:2.0:1761315666.652898:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567872 00000100:00100000:0.0:1761315666.652909:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4649 00000100:00100000:0.0:1761315666.652913:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3a868ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322567552:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315666.652926:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567488 00000100:00100000:2.0:1761315666.652930:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4650 00000100:00100000:1.0:1761315666.652934:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322567936:12345-0@lo:400:kworker/u8:2.0 Request processed in 70us (226us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315666.652940:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001014ca3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322567872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315666.652951:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1520 00000100:00100000:0.0:1761315666.652955:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3a868ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322567552:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315666.652956:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2904 00000100:00100000:0.0:1761315666.652966:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4649 00000100:00100000:3.0:1761315666.652966:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322568064:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315666.652970:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001014ca3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322567872:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (125us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315666.652977:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4650 00000004:00000024:3.0:1761315666.652994:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315666.653003:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315666.653007:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2905 00000100:00100000:2.0:1761315666.653011:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322567488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315666.653019:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568000:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315666.653027:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322567616 00000100:00100000:3.0:1761315666.653034:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322568064:12345-0@lo:41:osp-pre-0-1.0 Request processed in 69us (369us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315666.653041:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2904 00000100:00100000:0.0:1761315666.653046:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315666.653051:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322567488:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (273us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315666.653059:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2905 00000100:00100000:1.0:1761315666.653065:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2906 00000100:00100000:1.0:1761315666.653072:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9b923fa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322567616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315666.653074:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322567424:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315666.653088:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322567808:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.653090:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322567488:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.653101:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322567872:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315666.653103:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322568064:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315666.653110:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322568128:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315666.653117:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322567552:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315666.653121:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9b923fa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322567616:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (117us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315666.653130:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2906 00000100:00100000:0.0:1761315666.653131:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322567360:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.653131:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322567936:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315666.653147:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315666.653148:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322567616:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315666.653163:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568000 00000100:00100000:2.0:1761315666.653187:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568128 00000100:00100000:1.0:1761315666.653194:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1521 00000100:00100000:1.0:1761315666.653200:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315666.653230:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1009 00000100:00100000:2.0:1761315666.653234:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322568128:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315666.653248:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568000:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (203us total) trans 0 rc 0/0 00002000:00000024:2.0:1761315666.653254:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315666.653256:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1521 00000100:00100000:2.0:1761315666.653291:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322568128:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315666.653297:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1009 00000100:00100000:1.0:1761315666.653317:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322568128:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315666.653386:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568000:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761315667.675678:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315668.681761:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315668.681825:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560384 00000100:00100000:0.0:1761315668.681881:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2287 00000100:00100000:0.0:1761315668.681888:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307560384:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315668.681993:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315668.682062:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307560384:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 173us (308us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315668.682069:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2287 00000100:00100000:1.0:1761315668.684083:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315668.684133:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315668.684170:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315668.684413:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560320 00000100:00100000:2.0:1761315668.684477:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560256 00000100:00100000:2.0:1761315668.684500:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4651 00000100:00100000:2.0:1761315668.684507:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e34014aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307560320:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315668.684590:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e34014aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307560320:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 83us (458us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315668.684598:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4651 00000100:00100000:2.0:1761315668.684619:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4652 00000100:00100000:2.0:1761315668.684623:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307560256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315668.684642:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e25742f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307560256:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 19us (472us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315668.684648:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4652 00000100:00100000:0.0:1761315668.684886:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560192 00000100:00100000:0.0:1761315668.684925:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1719 00000100:00100000:0.0:1761315668.684929:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307560192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315668.684955:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307560192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (881us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315668.684961:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1719 00010000:00010000:2.0:1761315668.701322:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315669.213887:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322568192:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315669.213967:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315669.214124:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568192 00000100:00100000:1.0:1761315669.214184:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2907 00000100:00100000:1.0:1761315669.214189:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322568192:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315669.214228:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315669.214316:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322568192:12345-0@lo:41:osp-pre-1-0.0 Request processed in 126us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315669.214323:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2907 00000100:00100000:2.0:1761315669.214438:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322568192:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315669.725065:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315670.748972:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315671.772030:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322568256:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315671.772139:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.772248:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315671.772255:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322568320:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1761315671.772265:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315671.772285:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315671.772294:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568256 00000100:00100000:1.0:1761315671.772318:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568320 00000100:00100000:3.0:1761315671.772386:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2908 00000100:00100000:3.0:1761315671.772394:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322568256:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315671.772397:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1010 00000100:00100000:1.0:1761315671.772406:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322568320:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315671.772440:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000004:00000024:3.0:1761315671.772441:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:2.0:1761315671.772478:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315671.772485:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315671.772489:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315671.772495:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3914 00000100:00080000:2.0:1761315671.772499:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315671.772504:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315671.772524:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000077564dac x1846869322568640/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315671.772540:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315671.772543:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315671.772545:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315671.772548:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3914 00000100:00080000:2.0:1761315671.772550:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761315671.772552:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315671.772561:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f0419e0e x1846869322568704/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315671.772628:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322568320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 221us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315671.772638:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1010 00000100:00100000:0.0:1761315671.772649:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322568384:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.772673:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322568512:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.772699:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.772715:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315671.772724:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322568768:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.772738:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322568896:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.772754:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.772771:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315671.772783:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315671.772792:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322568448:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315671.772822:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568576:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.772824:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.772826:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a2ab8d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322568320:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315671.772849:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322568256:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315671.772853:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322568832:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315671.772869:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315671.772885:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.772900:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568512 00000100:00100000:2.0:1761315671.772911:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322568640:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315671.772930:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568384 00000100:00100000:1.0:1761315671.772938:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568768 00000100:00100000:2.0:1761315671.772944:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315671.772968:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322568704:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315671.772969:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568448 00000100:00100000:0.0:1761315671.772974:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1720 00000100:00100000:0.0:1761315671.772981:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322568384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.772990:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568832 00000100:00100000:2.0:1761315671.772999:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315671.773011:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4653 00000100:00100000:1.0:1761315671.773017:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322568512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.773035:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322568384:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (336us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315671.773044:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1720 00000100:00100000:0.0:1761315671.773078:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4654 00000100:00100000:0.0:1761315671.773082:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b566053 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322568768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.773110:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b566053 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322568768:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315671.773116:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4654 00000100:00100000:0.0:1761315671.773132:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4655 00000100:00100000:1.0:1761315671.773138:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3d6aa99 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322568512:12345-0@lo:400:kworker/u8:2.0 Request processed in 118us (421us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315671.773139:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006eaf6c19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322568448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773152:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4653 00000100:00100000:0.0:1761315671.773172:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006eaf6c19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322568448:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315671.773173:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4656 00000100:00100000:1.0:1761315671.773180:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000751f5dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322568832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315671.773184:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4655 00000100:00100000:0.0:1761315671.773220:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568576 00000100:00100000:1.0:1761315671.773223:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000751f5dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322568832:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315671.773230:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4656 00000100:00100000:1.0:1761315671.773256:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568896 00000100:00100000:1.0:1761315671.773283:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568640 00000100:00100000:1.0:1761315671.773297:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568704 00000100:00100000:1.0:1761315671.773311:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1522 00000100:00100000:1.0:1761315671.773315:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322568896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773364:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322568896:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (574us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315671.773370:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1522 00000100:00100000:1.0:1761315671.773391:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2288 00000100:00100000:1.0:1761315671.773394:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a67a644 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322568640:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315671.773487:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315671.773521:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a67a644 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322568640:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (578us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315671.773528:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2288 00000100:00100000:1.0:1761315671.773541:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2289 00000100:00100000:1.0:1761315671.773544:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322568704:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315671.773621:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315671.773646:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322568704:12345-0@lo:8:kworker/u8:2.0 Request processed in 100us (648us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315671.773651:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2289 00000100:00100000:3.0:1761315671.773671:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322568256:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1275us (1531us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315671.773681:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2908 00000100:00100000:3.0:1761315671.773705:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2909 00000100:00100000:3.0:1761315671.773709:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008c6ae23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773713:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322568512:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773727:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322568896:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315671.773753:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008c6ae23 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568576:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (886us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315671.773760:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2909 00000100:00100000:1.0:1761315671.773781:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322568448:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773795:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322568832:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773807:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322568384:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773827:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322568768:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315671.773827:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315671.773883:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315671.773889:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322568640:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315671.773908:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315671.773911:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315671.773914:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322568704:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315671.773975:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000088984be5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568576:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315672.797146:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315673.803050:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315673.803107:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315673.803140:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315673.803173:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315673.803221:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560640 00000100:00100000:3.0:1761315673.803283:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2290 00000100:00100000:3.0:1761315673.803291:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce505fe3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307560640:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:2.0:1761315673.803416:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560576 00010000:02020000:3.0:1761315673.803475:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315673.803555:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce505fe3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307560640:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 263us (380us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315673.803564:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2290 00000100:00100000:2.0:1761315673.803718:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560512 00000100:00100000:2.0:1761315673.803766:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4657 00000100:00100000:2.0:1761315673.803773:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a90ace4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307560576:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315673.803788:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4658 00000100:00100000:3.0:1761315673.803797:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005296ee71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307560512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315673.803846:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005296ee71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307560512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 50us (706us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315673.803853:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4658 00000100:00100000:2.0:1761315673.803892:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a90ace4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307560576:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 117us (783us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315673.803903:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4657 00000100:00100000:0.0:1761315673.803957:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560448 00000100:00100000:0.0:1761315673.804006:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1721 00000100:00100000:0.0:1761315673.804011:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307560448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315673.804048:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307560448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 36us (1011us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315673.804055:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1721 00010000:00010000:2.0:1761315673.821582:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315674.334818:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568960:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315674.334916:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315674.335551:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322568960 00000100:00100000:0.0:1761315674.335646:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2910 00000100:00100000:0.0:1761315674.335654:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568960:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315674.335714:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315674.337131:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322568960:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1468us (2209us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315674.337147:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2910 00000100:00100000:1.0:1761315674.337264:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322568960:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315674.849736:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315675.871625:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315676.891687:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315676.891698:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761315676.891703:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322569280:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315676.891704:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315676.891715:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3919 00000100:00080000:3.0:1761315676.891718:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315676.891725:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315676.891798:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315676.891848:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315676.891852:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002672a249 x1846869322569408/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315676.891856:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322569216:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315676.891876:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315676.891880:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315676.891882:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315676.891886:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3919 00000100:00080000:3.0:1761315676.891888:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761315676.891892:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315676.891895:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315676.891909:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000041f26507 x1846869322569472/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315676.891940:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569280 00000100:00100000:3.0:1761315676.891976:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd3bcd15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322569600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.891997:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4659 00000100:00100000:1.0:1761315676.892004:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322569280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315676.892015:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315676.892042:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315676.892049:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322569664:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892060:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000076c50cde pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322569280:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315676.892068:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4659 00000100:00100000:1.0:1761315676.892085:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569600 00000100:00100000:1.0:1761315676.892103:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4660 00000100:00100000:1.0:1761315676.892106:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322569600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892129:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322569600:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (117us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315676.892134:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4660 00000100:00100000:1.0:1761315676.892154:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569216 00000100:00100000:1.0:1761315676.892176:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2911 00000100:00100000:1.0:1761315676.892180:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322569216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315676.892199:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315676.892212:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322569216:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (321us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315676.892219:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2911 00000100:00100000:3.0:1761315676.892246:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd3bcd15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322569600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892256:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322569216:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892268:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322569280:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892385:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322569024:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315676.892420:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315676.892446:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322569088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892479:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315676.892480:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569024 00000100:00100000:2.0:1761315676.892503:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322569344:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892513:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1011 00000100:00100000:1.0:1761315676.892518:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5acab3d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322569024:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315676.892530:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315676.892546:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322569728:0@lo:400:kworker/u8:2.0 00002000:00000024:1.0:1761315676.892548:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315676.892575:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315676.892588:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5acab3d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322569024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (167us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315676.892596:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1011 00000100:00100000:1.0:1761315676.892617:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322569408:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892622:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322569024:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315676.892636:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315676.892645:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322569152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315676.892649:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315676.892672:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041f26507 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322569472:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892681:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315676.892697:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322569536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892716:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315676.892744:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569344 00000100:00100000:2.0:1761315676.892767:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1761315676.892771:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322569344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892803:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322569344:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (275us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315676.892809:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2912 00000100:00100000:2.0:1761315676.892826:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569152 00000100:00100000:2.0:1761315676.892846:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569536 00000100:00100000:2.0:1761315676.892863:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4661 00000100:00100000:2.0:1761315676.892867:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000098941308 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322569152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892896:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000098941308 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322569152:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (216us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315676.892902:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4661 00000100:00100000:2.0:1761315676.892913:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4662 00000100:00100000:2.0:1761315676.892917:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000068985f9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322569536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.892942:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000068985f9b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322569536:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315676.892948:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4662 00000100:00100000:2.0:1761315676.892971:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569088 00000100:00100000:2.0:1761315676.892994:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1722 00000100:00100000:2.0:1761315676.892998:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322569088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893027:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322569088:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (548us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315676.893033:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1722 00000100:00100000:3.0:1761315676.893232:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569664 00000100:00100000:3.0:1761315676.893264:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569728 00000100:00100000:3.0:1761315676.893282:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569408 00000100:00100000:3.0:1761315676.893300:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1523 00000100:00100000:3.0:1761315676.893305:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322569664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315676.893368:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322569664:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (1171us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315676.893374:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1523 00000100:00100000:3.0:1761315676.893386:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1524 00000100:00100000:3.0:1761315676.893390:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038a7bd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322569728:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315676.893414:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038a7bd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322569728:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (841us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315676.893419:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1524 00000100:00100000:3.0:1761315676.893430:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2291 00000100:00100000:3.0:1761315676.893433:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322569408:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315676.893533:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315676.893564:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322569408:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (918us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315676.893570:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2291 00000100:00100000:3.0:1761315676.893621:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322569664:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893901:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322569152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893916:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322569536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893926:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322569088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893936:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322569344:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315676.893945:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322569728:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315676.896533:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1761315676.896560:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315676.896560:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e29b36b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322569792:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315676.896655:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315676.896888:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569792 00000100:00100000:0.0:1761315676.896951:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2913 00000100:00100000:0.0:1761315676.896958:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322569792:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315676.897001:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315676.897038:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315676.897085:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322569792:12345-0@lo:41:osp-pre-0-1.0 Request processed in 125us (430us total) trans 0 rc 0/0 00000100:00080000:1.0:1761315676.897086:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315676.897092:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315676.897097:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2913 00000100:00100000:1.0:1761315676.897099:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322569408:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315676.897136:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569472 00000100:00100000:0.0:1761315676.897145:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e29b36b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322569792:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315676.897164:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2292 00000100:00100000:1.0:1761315676.897169:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322569472:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315676.897269:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315676.897320:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322569472:12345-0@lo:8:kworker/u8:2.0 Request processed in 150us (283us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315676.897326:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2292 00000100:00080000:0.0:1761315676.901285:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315676.901299:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315676.901310:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041f26507 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322569472:0@lo:8:kworker/u8:2.0 00010000:00010000:2.0:1761315677.917163:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315678.922448:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315678.922495:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315678.922533:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315678.922622:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315678.922641:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560832 00000100:00100000:3.0:1761315678.922670:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560896 00000100:00100000:3.0:1761315678.922733:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2293 00000100:00100000:2.0:1761315678.922736:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560768 00000100:00100000:3.0:1761315678.922744:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307560896:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:2.0:1761315678.922773:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4663 00000100:00100000:2.0:1761315678.922781:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307560832:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00010000:02020000:3.0:1761315678.922846:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315678.922895:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307560832:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 111us (399us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315678.922905:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4663 00000100:00100000:3.0:1761315678.922916:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307560896:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 171us (295us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315678.922928:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2293 00000100:00100000:2.0:1761315678.922937:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4664 00000100:00100000:2.0:1761315678.922964:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307560768:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315678.923008:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307560768:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 46us (474us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315678.923017:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4664 00000100:00100000:0.0:1761315678.924570:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560704 00000100:00100000:0.0:1761315678.924645:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1723 00000100:00100000:0.0:1761315678.924652:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307560704:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315678.924736:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307560704:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 83us (2296us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315678.924744:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1723 00010000:00010000:3.0:1761315678.941926:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315679.451694:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315679.451837:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322569856:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315679.451931:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315679.452050:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569856 00000100:00100000:3.0:1761315679.452117:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2914 00000100:00100000:3.0:1761315679.452125:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322569856:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315679.452172:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315679.453706:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322569856:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1574us (1769us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315679.453723:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2914 00000100:00100000:1.0:1761315679.453848:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322569856:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315679.964858:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315680.987606:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315682.011550:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315682.011557:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315682.011561:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315682.011569:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3924 00000100:00080000:3.0:1761315682.011572:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315682.011578:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315682.011598:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000004b40748 x1846869322570240/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315682.011626:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315682.011628:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315682.011630:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315682.011634:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3924 00000100:00080000:3.0:1761315682.011635:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315682.011638:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315682.011649:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000003430de1 x1846869322570304/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315682.012029:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322569920:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.012132:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315682.012185:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322570176:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.012216:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315682.012234:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002df1e9e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322570560:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.012259:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315682.012285:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315682.012292:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322570112:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.012371:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315682.012388:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322570496:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.012410:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315682.012567:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569920 00000100:00100000:0.0:1761315682.012594:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322570240:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315682.012637:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315682.012646:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1724 00000100:00100000:3.0:1761315682.012655:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322569920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315682.012665:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322570304:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315682.012706:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315682.012764:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322569920:12345-0@lo:400:kworker/u8:2.0 Request processed in 108us (632us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315682.012765:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570560 00000100:00100000:3.0:1761315682.012776:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1724 00000100:00100000:0.0:1761315682.012817:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570496 00000100:00100000:3.0:1761315682.012835:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570240 00000100:00100000:0.0:1761315682.012847:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570304 00000100:00100000:3.0:1761315682.012861:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1525 00000100:00100000:3.0:1761315682.012868:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322570560:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315682.012904:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322570560:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (645us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315682.013018:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322569920:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.013037:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002df1e9e0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322570560:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315682.013203:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570048:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315682.013246:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315682.013268:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322570432:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315682.013291:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315682.013321:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1761315682.013327:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322569984:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315682.013369:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315682.013386:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322570368:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315682.013409:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315682.013423:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570624:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315682.013442:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315682.013555:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1526 00000100:00100000:0.0:1761315682.013562:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322570496:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315682.013611:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322570496:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (1200us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315682.013618:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1526 00000100:00100000:0.0:1761315682.013635:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2294 00000100:00100000:0.0:1761315682.013639:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e29b36b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322570240:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315682.013725:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1525 00010000:02020000:0.0:1761315682.013736:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315682.013754:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2295 00000100:00100000:3.0:1761315682.013761:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc23aeea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322570304:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315682.013777:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e29b36b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322570240:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (1140us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315682.013786:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2294 00000100:00100000:0.0:1761315682.013816:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570176 00000100:00100000:0.0:1761315682.013844:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570048 00000100:00100000:0.0:1761315682.013854:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570624 00010000:02020000:3.0:1761315682.013860:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315682.013868:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1761315682.013874:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322570176:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315682.013923:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc23aeea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322570304:12345-0@lo:8:kworker/u8:2.0 Request processed in 158us (1215us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315682.013930:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322570176:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (1711us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315682.013936:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2295 00000100:00100000:0.0:1761315682.013953:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2915 00000100:00100000:0.0:1761315682.013983:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2916 00000100:00100000:0.0:1761315682.013993:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322570048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315682.013996:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2917 00000100:00100000:3.0:1761315682.014009:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322570624:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315682.014043:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315682.014049:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322570048:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (801us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315682.014059:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2916 00000100:00100000:3.0:1761315682.014098:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322570624:12345-0@lo:41:osp-pre-0-1.0 Request processed in 88us (652us total) trans 0 rc 0/0 00000100:00080000:0.0:1761315682.014106:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761315682.014109:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2917 00000100:00080000:0.0:1761315682.014113:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315682.014120:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322570240:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315682.014140:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315682.014143:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315682.014160:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322570304:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315682.014236:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570112 00000100:00100000:3.0:1761315682.014279:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570432 00010000:00010000:0.0:1761315682.014285:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315682.014302:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b7d9eb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570688:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315682.014316:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322569984 00000100:00100000:3.0:1761315682.014339:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570368 00000100:00100000:0.0:1761315682.014395:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315682.014404:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4665 00000100:00100000:3.0:1761315682.014411:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322570112:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315682.014439:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570624:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315682.014456:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315682.014483:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000088984be5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322570112:12345-0@lo:400:kworker/u8:2.0 Request processed in 69us (2109us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315682.014490:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4665 00000100:00100000:0.0:1761315682.014525:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4666 00000100:00100000:1.0:1761315682.014652:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322570112:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.014662:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322570496:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.014673:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322570176:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.014720:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570688 00000100:00100000:1.0:1761315682.014745:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1012 00000100:00100000:1.0:1761315682.014749:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322570688:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315682.014770:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315682.014805:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322570688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315682.014811:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1012 00000100:00100000:1.0:1761315682.014843:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b7d9eb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570688:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315682.014962:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000948a8d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322570432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315682.015001:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000948a8d09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322570432:12345-0@lo:400:kworker/u8:2.0 Request processed in 469us (1710us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315682.015469:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4667 00000100:00100000:3.0:1761315682.015478:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cdccde7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322569984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.015485:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4668 00000100:00100000:1.0:1761315682.015496:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b44ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322570368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315682.015553:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cdccde7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322569984:12345-0@lo:400:kworker/u8:2.0 Request processed in 75us (2182us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315682.015561:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b44ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322570368:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (2149us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315682.015568:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4667 00000100:00100000:1.0:1761315682.015580:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4668 00000100:00100000:1.0:1761315682.015645:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322569984:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.015655:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322570368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315682.015663:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322570432:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315682.016298:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4666 00010000:00010000:3.0:1761315683.036249:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315684.041495:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315684.042583:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315684.043426:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561024 00000100:00100000:0.0:1761315684.043480:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4669 00000100:00100000:0.0:1761315684.043488:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f0aeaf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307561024:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315684.043556:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f0aeaf6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307561024:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 68us (2068us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315684.043563:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4669 00000100:00100000:3.0:1761315684.047112:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315684.047161:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315684.047486:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307560960 00000100:00100000:1.0:1761315684.047535:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1725 00000100:00100000:1.0:1761315684.047542:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307560960:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315684.047605:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307560960:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 62us (499us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315684.047616:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1725 00000100:00100000:0.0:1761315684.047968:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561088 00000100:00100000:0.0:1761315684.048014:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4670 00000100:00100000:0.0:1761315684.048019:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307561088:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315684.048052:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002df1e9e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307561088:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (892us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315684.048058:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4670 00000100:00100000:2.0:1761315684.049392:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561152 00000100:00100000:2.0:1761315684.049445:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2296 00000100:00100000:2.0:1761315684.049450:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307561152:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315684.049541:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315684.049604:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307561152:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 152us (7022us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315684.049611:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2296 00010000:00010000:2.0:1761315684.059493:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315684.571653:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322570752:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315684.571724:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315684.571883:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570752 00000100:00100000:3.0:1761315684.571932:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2918 00000100:00100000:3.0:1761315684.571938:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322570752:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315684.571970:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315684.572028:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322570752:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (304us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315684.572035:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2918 00000100:00100000:0.0:1761315684.572163:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322570752:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315685.083498:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315686.108661:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315687.132589:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315687.132710:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322570816:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315687.132786:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.133103:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570816 00000100:00100000:0.0:1761315687.133143:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2919 00000100:00100000:0.0:1761315687.133148:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061659632 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322570816:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315687.133178:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315687.134461:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061659632 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322570816:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1308us (1673us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.134471:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2919 00000100:00100000:2.0:1761315687.135745:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322570816:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315687.139168:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315687.139728:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322570944:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315687.139796:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315687.139899:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315687.139903:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315687.139906:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315687.139911:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3929 00000100:00080000:1.0:1761315687.139913:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315687.139917:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315687.139933:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e334a674 x1846869322571200/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315687.139946:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315687.139949:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315687.139950:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315687.139953:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3929 00000100:00080000:1.0:1761315687.139955:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315687.139957:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315687.139964:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004c227bf8 x1846869322571264/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315687.140004:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322571008:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315687.140034:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315687.140055:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322571392:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140419:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322571136:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140451:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.140466:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322571520:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140489:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.140511:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315687.140515:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322571328:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140533:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.140561:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571008 00000100:00100000:0.0:1761315687.140597:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571328 00000100:00100000:0.0:1761315687.140618:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4671 00000100:00100000:0.0:1761315687.140623:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004b40748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322571008:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140660:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004b40748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322571008:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (626us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.140667:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4671 00000100:00100000:0.0:1761315687.140682:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4672 00000100:00100000:0.0:1761315687.140685:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000050474828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322571328:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140710:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000050474828 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322571328:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.140715:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4672 00000100:00100000:0.0:1761315687.140747:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322571328:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140777:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571136 00000100:00100000:0.0:1761315687.140798:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2920 00000100:00100000:0.0:1761315687.140801:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322571136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140844:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322571136:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (394us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.140849:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2920 00000100:00100000:0.0:1761315687.140867:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322571200:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140891:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.140904:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322571264:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315687.140921:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.140947:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322571136:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141004:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571520 00000100:00100000:0.0:1761315687.141026:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571200 00000100:00100000:0.0:1761315687.141036:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571264 00000100:00100000:0.0:1761315687.141046:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1527 00000100:00100000:0.0:1761315687.141049:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322571520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141078:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322571520:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (589us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.141084:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1527 00000100:00100000:0.0:1761315687.141093:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2297 00000100:00100000:0.0:1761315687.141096:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322571200:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315687.141182:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315687.141209:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322571200:12345-0@lo:8:kworker/u8:0.0 Request processed in 111us (318us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315687.141215:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2297 00000100:00100000:0.0:1761315687.141224:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2298 00000100:00100000:0.0:1761315687.141227:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322571264:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315687.141317:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315687.141337:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322571264:12345-0@lo:8:kworker/u8:0.0 Request processed in 109us (416us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315687.141366:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2298 00000100:00100000:0.0:1761315687.141392:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322571520:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315687.141411:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315687.141414:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315687.141416:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322571200:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315687.141423:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315687.141425:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315687.141427:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322571264:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141484:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322571072:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141507:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.141519:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322571456:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141535:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315687.141554:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571456 00000100:00100000:0.0:1761315687.141571:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1528 00000100:00100000:0.0:1761315687.141574:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322571456:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141604:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322571456:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (69us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.141610:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1528 00000100:00100000:0.0:1761315687.141624:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571072 00000100:00100000:0.0:1761315687.141639:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4673 00000100:00100000:0.0:1761315687.141642:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322571072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141666:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007555ef20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322571072:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (159us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.141671:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4673 00000100:00100000:0.0:1761315687.141694:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322571072:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315687.141702:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322571456:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315687.142614:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315687.142647:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315687.143795:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315687.143843:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322571008:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315687.144101:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570944 00000100:00100000:3.0:1761315687.144151:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1726 00000100:00100000:3.0:1761315687.144156:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322570944:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315687.144209:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322570944:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (4413us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315687.144217:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1726 00000100:00100000:0.0:1761315687.144378:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322570880 00000100:00100000:0.0:1761315687.144414:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1013 00000100:00100000:0.0:1761315687.144418:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322570880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315687.144440:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315687.144473:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322570880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (1825us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315687.144478:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1013 00000100:00100000:2.0:1761315687.144607:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571392 00000100:00100000:2.0:1761315687.144644:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4674 00000100:00100000:2.0:1761315687.144648:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6aa85f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322571392:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315687.144695:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6aa85f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322571392:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (902us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315687.144701:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4674 00000100:00100000:2.0:1761315687.144731:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322570944:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315687.144748:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322570880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315687.144826:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322571392:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315688.158525:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315689.167586:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315689.167630:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315689.167657:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315689.167689:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315689.167769:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561216 00000100:00100000:0.0:1761315689.167820:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1727 00000100:00100000:0.0:1761315689.167826:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307561216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315689.167886:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307561216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 60us (308us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315689.167894:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1727 00000100:00100000:3.0:1761315689.168080:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561344 00000100:00100000:3.0:1761315689.168124:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561280 00000100:00100000:3.0:1761315689.168153:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4675 00000100:00100000:3.0:1761315689.168157:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba19120 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307561344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315689.168401:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4676 00000100:00100000:1.0:1761315689.168406:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307561280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315689.168441:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307561280:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (784us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315689.168448:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4676 00000100:00100000:1.0:1761315689.168485:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561408 00000100:00100000:1.0:1761315689.168503:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2299 00000100:00100000:1.0:1761315689.168507:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307561408:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315689.168597:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315689.168624:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307561408:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 116us (935us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315689.168631:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2299 00000100:00100000:3.0:1761315689.168928:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba19120 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307561344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 769us (1298us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315689.168936:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4675 00010000:00010000:3.0:1761315689.179643:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315689.692560:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322571584:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315689.692633:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315689.695821:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571584 00000100:00100000:0.0:1761315689.698631:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2921 00000100:00100000:0.0:1761315689.698643:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322571584:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315689.698687:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315689.698777:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322571584:12345-0@lo:41:osp-pre-1-0.0 Request processed in 134us (6144us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315689.698785:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2921 00000100:00100000:2.0:1761315689.698886:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322571584:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315690.203682:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315691.227506:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315692.251533:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322571648:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315692.251550:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315692.251555:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315692.251559:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315692.251564:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3934 00000100:00080000:1.0:1761315692.251567:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315692.251573:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315692.251594:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005a3d67ed x1846869322571968/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315692.251607:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315692.251608:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315692.251612:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315692.251614:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315692.251617:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3934 00000100:00080000:1.0:1761315692.251619:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761315692.251623:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315692.251633:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000077564dac x1846869322572032/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315692.251642:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322571904:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251675:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315692.251699:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322572288:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.251703:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322571776:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251733:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315692.251748:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315692.251756:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315692.251763:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322571712:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.251774:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322572160:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251796:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315692.251823:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571904 00000100:00100000:2.0:1761315692.251823:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322572096:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251851:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315692.251867:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571776 00000100:00100000:2.0:1761315692.251879:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322571840:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.251895:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2922 00000100:00100000:1.0:1761315692.251901:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322571904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251907:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315692.251926:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322572224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251955:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315692.251964:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322571904:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.251972:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2922 00000100:00100000:1.0:1761315692.251993:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2923 00000100:00100000:1.0:1761315692.251997:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322571776:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.251997:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572288 00000100:00100000:2.0:1761315692.252028:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572224 00000100:00100000:1.0:1761315692.252029:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322571776:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.252038:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2923 00000100:00100000:2.0:1761315692.252051:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1529 00000100:00100000:2.0:1761315692.252057:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bca5f7bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322572288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.252067:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1530 00000100:00100000:1.0:1761315692.252073:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322572224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315692.252096:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bca5f7bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322572288:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (363us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315692.252104:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1529 00000100:00100000:1.0:1761315692.252118:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322572224:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.252124:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1530 00000100:00100000:2.0:1761315692.252132:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571712 00000100:00100000:1.0:1761315692.252149:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315692.253114:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322572352:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315692.253151:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315692.253187:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322571968:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315692.253224:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315692.253249:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322572032:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315692.253268:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315692.253296:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571968 00000100:00100000:0.0:1761315692.253318:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572032 00000100:00100000:0.0:1761315692.253330:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2300 00000100:00100000:0.0:1761315692.253334:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322571968:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315692.253435:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315692.253471:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322571968:12345-0@lo:8:kworker/u8:0.0 Request processed in 135us (246us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315692.253477:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2300 00000100:00100000:0.0:1761315692.253491:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2301 00000100:00100000:0.0:1761315692.253494:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322572032:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315692.253575:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315692.253595:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322572032:12345-0@lo:8:kworker/u8:0.0 Request processed in 100us (326us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315692.253600:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2301 00000100:00100000:0.0:1761315692.253679:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572352 00000100:00100000:0.0:1761315692.253707:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1014 00000100:00100000:0.0:1761315692.253711:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322572352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315692.253734:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315692.253768:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322572352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (616us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315692.253773:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1014 00000100:00100000:0.0:1761315692.253821:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322572352:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315692.253852:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315692.253857:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315692.253860:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322571968:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315692.253868:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315692.253870:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315692.253872:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322572032:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315692.253907:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571648 00000100:00100000:0.0:1761315692.253930:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1728 00000100:00100000:0.0:1761315692.253933:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322571648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315692.254654:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322571840 00000100:00100000:2.0:1761315692.254658:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572096 00000100:00100000:3.0:1761315692.254702:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572160 00000100:00100000:2.0:1761315692.254706:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4677 00000100:00100000:2.0:1761315692.254713:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322571712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315692.254722:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4678 00000100:00100000:3.0:1761315692.254743:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322572096:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256013:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322572416:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315692.256064:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315692.256095:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322571776:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256137:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4679 00000100:00100000:1.0:1761315692.256142:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a67a644 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322571840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256178:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a67a644 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322571840:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (4274us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.256186:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4679 00000100:00100000:1.0:1761315692.256218:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4680 00000100:00100000:1.0:1761315692.256222:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322572160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256262:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322572160:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (4114us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.256266:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4680 00000100:00100000:1.0:1761315692.256284:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572416 00000100:00100000:1.0:1761315692.256307:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2924 00000100:00100000:1.0:1761315692.256310:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322572416:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315692.256331:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315692.256384:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322572416:12345-0@lo:41:osp-pre-0-1.0 Request processed in 72us (320us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315692.256389:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2924 00000100:00100000:1.0:1761315692.256420:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322572288:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256429:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322571904:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256437:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322571648:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256474:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322572416:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315692.256483:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322572160:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256549:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322571840:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315692.256559:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001945c9c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322572224:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315692.256590:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315692.257475:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322571712:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315692.257558:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000085b3c147 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322572096:12345-0@lo:400:kworker/u8:0.0 Request processed in 2828us (5706us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315692.257566:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4678 00000100:00100000:1.0:1761315692.257637:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001eebd285 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322572096:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315692.258077:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322571648:12345-0@lo:400:kworker/u8:0.0 Request processed in 4142us (6472us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315692.258085:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1728 00000100:00100000:2.0:1761315692.258199:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2bf2d15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322571712:12345-0@lo:400:kworker/u8:0.0 Request processed in 3485us (6403us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315692.258207:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4677 00010000:00010000:0.0:1761315693.275585:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315694.280760:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315694.280881:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561472 00000100:00100000:2.0:1761315694.280951:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1729 00000100:00100000:2.0:1761315694.280959:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307561472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315694.281039:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307561472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 80us (292us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315694.281046:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1729 00000100:00100000:1.0:1761315694.283647:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315694.283694:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315694.283727:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315694.283836:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561536 00000100:00100000:0.0:1761315694.283857:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561664 00000100:00100000:0.0:1761315694.283915:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2302 00000100:00100000:0.0:1761315694.283924:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307561664:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315694.284047:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315694.284677:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561600 00000100:00100000:1.0:1761315694.284727:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4681 00000100:00100000:1.0:1761315694.284733:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307561536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315694.284857:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4682 00000100:00100000:3.0:1761315694.284866:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307561600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315694.288309:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307561536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 3567us (4607us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315694.288325:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4681 00000100:00100000:0.0:1761315694.288913:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307561664:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 4988us (5273us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315694.288925:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2302 00000100:00100000:3.0:1761315694.288981:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307561600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 4106us (5246us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315694.288998:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4682 00010000:00010000:0.0:1761315694.299501:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315694.811746:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315694.811776:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322572480:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315694.811876:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315694.811999:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572480 00000100:00100000:3.0:1761315694.812077:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2925 00000100:00100000:3.0:1761315694.812085:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322572480:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315694.812132:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315694.812207:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322572480:12345-0@lo:41:osp-pre-1-0.0 Request processed in 122us (332us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315694.812242:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2925 00000100:00100000:2.0:1761315694.812337:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322572480:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315695.324309:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315696.347614:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315697.371689:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315697.371694:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322572672:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315697.371699:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315697.371704:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315697.371712:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3939 00000100:00080000:3.0:1761315697.371715:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315697.371723:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315697.371749:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d5431e2e x1846869322572928/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315697.371767:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315697.371771:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315697.371773:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315697.371776:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3939 00000100:00080000:3.0:1761315697.371778:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761315697.371781:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315697.371796:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315697.371797:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000be19034 x1846869322572992/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315697.371853:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322573056:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315697.371862:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322572608:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315697.371888:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315697.371912:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315697.371913:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315697.371921:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322572736:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315697.371941:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c3594b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322572864:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315697.371951:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315697.371972:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322573120:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315697.371978:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315697.372000:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322573248:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315697.372007:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315697.372036:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315697.372055:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572608 00000100:00100000:1.0:1761315697.372104:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572736 00000100:00100000:1.0:1761315697.372116:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572864 00000100:00100000:2.0:1761315697.372126:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322572544:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315697.372134:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2926 00000100:00100000:1.0:1761315697.372141:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000012cee7c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322572608:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315697.372175:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:1.0:1761315697.372188:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315697.372207:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322572800:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372240:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315697.372263:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b623220c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322573184:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372289:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315697.372323:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572672 00000100:00100000:2.0:1761315697.372397:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573056 00000100:00100000:2.0:1761315697.372416:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573120 00000100:00100000:2.0:1761315697.372427:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572800 00000100:00100000:2.0:1761315697.372438:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4683 00000100:00100000:2.0:1761315697.372442:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322572672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372483:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001eebd285 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322572672:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (691us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315697.372491:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4683 00000100:00100000:2.0:1761315697.372509:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4684 00000100:00100000:2.0:1761315697.372513:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322573056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372544:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001945c9c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322573056:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (658us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315697.372549:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4684 00000100:00100000:2.0:1761315697.372560:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4685 00000100:00100000:2.0:1761315697.372564:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322573120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372586:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322573120:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (581us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315697.372590:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4685 00000100:00100000:2.0:1761315697.372597:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4686 00000100:00100000:2.0:1761315697.372600:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322572800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315697.372619:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055861db2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322572800:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (380us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315697.372624:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4686 00000100:00100000:2.0:1761315697.372643:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573248 00000100:00100000:2.0:1761315697.372662:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573184 00000100:00100000:2.0:1761315697.372675:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1531 00000100:00100000:2.0:1761315697.372677:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bdc06b77 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322573248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.373667:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1532 00000100:00100000:0.0:1761315697.373677:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322573184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.373751:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322573184:12345-0@lo:400:kworker/u8:2.0 Request processed in 76us (1462us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315697.373759:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1532 00010000:00010000:0.0:1761315697.373795:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315697.373866:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315697.373877:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000956eaacd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322573312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315697.373913:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315697.373946:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573312 00000100:00100000:0.0:1761315697.373975:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1015 00000100:00100000:0.0:1761315697.373979:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322573312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315697.374001:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315697.374031:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322573312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (119us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315697.374037:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1015 00000100:00100000:0.0:1761315697.374080:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000956eaacd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322573312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315697.374124:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2927 00000100:00100000:0.0:1761315697.374128:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322572736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374160:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322572736:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (2211us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315697.374166:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2927 00000100:00100000:0.0:1761315697.374177:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2928 00000100:00100000:0.0:1761315697.374181:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322572864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374219:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322572864:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (2245us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315697.374224:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2928 00000100:00100000:0.0:1761315697.374246:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572544 00000100:00100000:0.0:1761315697.374274:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1730 00000100:00100000:0.0:1761315697.374278:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322572544:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374309:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322572544:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (2135us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315697.374314:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1730 00000100:00100000:0.0:1761315697.374420:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000354757ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322572544:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374437:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000007e02a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322572800:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374445:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b623220c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322573184:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374484:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322572736:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374495:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000046b7f05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322573120:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374506:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322573056:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.374517:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c45a5c1f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322572672:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315697.375262:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322572928:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315697.375738:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315697.375771:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322572992:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315697.375795:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315697.375843:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572928 00000100:00100000:0.0:1761315697.375873:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322572992 00000100:00100000:0.0:1761315697.375889:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2303 00000100:00100000:0.0:1761315697.375894:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322572928:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315697.375991:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315697.376036:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322572928:12345-0@lo:8:kworker/u8:2.0 Request processed in 141us (300us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315697.376044:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2303 00000100:00100000:0.0:1761315697.376060:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2304 00000100:00100000:0.0:1761315697.376063:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322572992:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315697.376135:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315697.376158:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322572992:12345-0@lo:8:kworker/u8:2.0 Request processed in 94us (363us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315697.376165:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2304 00000100:00080000:0.0:1761315697.376223:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315697.376229:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315697.376234:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5431e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322572928:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315697.376244:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315697.376246:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315697.376249:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000be19034 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322572992:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315697.378833:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bdc06b77 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322573248:12345-0@lo:400:kworker/u8:2.0 Request processed in 6141us (6786us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315697.378854:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1531 00000100:00100000:1.0:1761315697.379138:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000012cee7c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322572608:12345-0@lo:41:osp-pre-0-1.0 Request processed in 6994us (7225us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315697.379148:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2926 00000100:00100000:3.0:1761315697.379272:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322572608:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315697.379304:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c3594b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322572864:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315697.379331:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322573248:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315698.395644:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315699.401955:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315699.402013:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315699.402130:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561792 00000100:00100000:2.0:1761315699.402201:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4687 00000100:00100000:2.0:1761315699.402208:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307561792:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315699.402282:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000046b7f05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307561792:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 73us (334us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315699.402290:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4687 00000100:00100000:1.0:1761315699.403317:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561728 00000100:00100000:1.0:1761315699.403375:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1731 00000100:00100000:1.0:1761315699.403381:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307561728:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315699.403421:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307561728:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 40us (1408us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315699.403428:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1731 00000100:00100000:0.0:1761315699.406842:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315699.407013:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561856 00000100:00100000:3.0:1761315699.407068:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4688 00000100:00100000:3.0:1761315699.407075:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307561856:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315699.407154:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b623220c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307561856:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 79us (318us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315699.407161:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4688 00000100:00100000:2.0:1761315699.410899:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315699.411501:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561920 00000100:00100000:0.0:1761315699.411555:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2305 00000100:00100000:0.0:1761315699.411561:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307561920:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315699.411658:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315699.411731:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307561920:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 169us (840us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315699.411739:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2305 00010000:00010000:0.0:1761315699.419681:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315699.931629:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315699.931647:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000956dea7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322573376:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315699.931723:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315699.931837:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573376 00000100:00100000:1.0:1761315699.931886:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2929 00000100:00100000:1.0:1761315699.931891:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322573376:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315699.931924:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315699.931980:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322573376:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (257us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315699.931987:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2929 00000100:00100000:3.0:1761315699.932071:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000956dea7b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322573376:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315700.443681:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315701.467496:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315702.492194:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315702.500333:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315702.500377:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322573440:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315702.500449:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.500502:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573440 00000100:00100000:3.0:1761315702.500550:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1016 00000100:00100000:3.0:1761315702.500556:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322573440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315702.500587:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315702.500626:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322573440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (178us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315702.500632:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1016 00000100:00100000:3.0:1761315702.500678:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322573440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315702.500886:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322573568:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315702.500899:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315702.500905:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315702.500909:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315702.500914:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3944 00000100:00080000:1.0:1761315702.500917:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315702.500922:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315702.500926:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.500988:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322573504:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315702.501011:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322573696:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501018:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.501037:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322573760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.501051:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.501067:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315702.501098:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573504 00000100:00100000:3.0:1761315702.501111:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573760 00000100:00100000:2.0:1761315702.501131:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573696 00000100:00100000:3.0:1761315702.501146:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4689 00000100:00100000:3.0:1761315702.501151:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4fdb62e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322573760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.501169:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2930 00000100:00100000:2.0:1761315702.501175:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a986d022 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322573504:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315702.501200:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315702.501200:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4fdb62e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322573760:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (133us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315702.501210:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4689 00000100:00100000:2.0:1761315702.501244:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a986d022 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322573504:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (225us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315702.501248:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004efff1f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322573504:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315702.501254:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2930 00000100:00100000:3.0:1761315702.501263:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322573760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.501274:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2931 00000100:00100000:2.0:1761315702.501279:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322573696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501308:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573568 00000100:00100000:2.0:1761315702.501316:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322573696:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (266us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315702.501323:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2931 00000100:00100000:3.0:1761315702.501338:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1732 00000100:00100000:3.0:1761315702.501366:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322573568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.501393:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322573696:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501413:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322573568:12345-0@lo:400:kworker/u8:0.0 Request processed in 69us (487us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315702.501415:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322573632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501426:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1732 00000100:00100000:2.0:1761315702.501453:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.501473:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573632 00000100:00100000:2.0:1761315702.501476:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315702.501484:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322573824:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501497:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4690 00000100:00100000:3.0:1761315702.501502:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e896943 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322573632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.501511:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.501540:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e896943 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322573632:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (88us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315702.501547:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4690 00000100:00100000:2.0:1761315702.501550:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322573632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501567:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573824 00000100:00100000:3.0:1761315702.501597:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2932 00000100:00100000:3.0:1761315702.501602:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322573824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.501633:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322573824:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (124us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315702.501640:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2932 00000100:00100000:2.0:1761315702.501657:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322573824:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315702.502507:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322573568:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315702.503068:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d6897f1b x1846869322573888/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315702.503098:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315702.503102:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315702.503104:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315702.503107:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3944 00000100:00080000:1.0:1761315702.503109:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315702.503114:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315702.503134:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cf0de5dc x1846869322573952/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315702.503204:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322574016:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315702.503242:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.503305:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574016 00000100:00100000:0.0:1761315702.503324:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322573888:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315702.503362:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4691 00000100:00100000:3.0:1761315702.503369:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322574016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315702.503379:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315702.503398:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf0de5dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322573952:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315702.503433:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315702.503462:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573888 00000100:00100000:0.0:1761315702.503492:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322573952 00000100:00100000:0.0:1761315702.503513:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2306 00000100:00100000:0.0:1761315702.503517:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322573888:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315702.503609:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315702.503655:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322573888:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (278us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315702.503666:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2306 00000100:00100000:0.0:1761315702.503684:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2307 00000100:00100000:0.0:1761315702.503687:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322573952:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315702.503782:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315702.503814:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322573952:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (381us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315702.503825:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2307 00000100:00080000:0.0:1761315702.503874:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315702.503878:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315702.503885:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322573888:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315702.504488:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315702.504491:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315702.504495:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf0de5dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322573952:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315702.505231:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322574080:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315702.505273:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315702.505295:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315702.505300:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322574144:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315702.505321:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315702.505569:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000955fb331 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322574016:12345-0@lo:400:kworker/u8:0.0 Request processed in 2198us (2324us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315702.505572:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322574016:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.505582:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4691 00000100:00100000:0.0:1761315702.505664:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574080 00000100:00100000:0.0:1761315702.505745:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4692 00000100:00100000:0.0:1761315702.505751:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c97fc10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322574080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315702.506324:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c97fc10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322574080:12345-0@lo:400:kworker/u8:0.0 Request processed in 573us (1050us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315702.506331:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4692 00000100:00100000:1.0:1761315702.506417:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574144 00000100:00100000:1.0:1761315702.506454:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1533 00000100:00100000:1.0:1761315702.506459:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322574144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315702.506490:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322574144:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (1168us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315702.506496:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1533 00000100:00100000:3.0:1761315702.506600:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322574080:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315702.506619:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322574144:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315703.515685:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315704.521801:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315704.521853:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315704.521890:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315704.521932:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315704.522023:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562048 00000100:00100000:1.0:1761315704.522079:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562112 00000100:00100000:1.0:1761315704.522099:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4693 00000100:00100000:1.0:1761315704.522105:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315704.522182:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000be19034 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 77us (390us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315704.522190:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4693 00000100:00100000:1.0:1761315704.522206:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4694 00000100:00100000:1.0:1761315704.522210:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c45a5c1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307562112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315704.522234:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c45a5c1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307562112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315704.522239:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4694 00000100:00100000:3.0:1761315704.522487:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562176 00000100:00100000:3.0:1761315704.522520:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2308 00000100:00100000:3.0:1761315704.522524:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307562176:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315704.522619:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315704.522649:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307562176:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 123us (759us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315704.522656:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2308 00000100:00100000:2.0:1761315704.522874:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307561984 00000100:00100000:2.0:1761315704.522909:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1733 00000100:00100000:2.0:1761315704.522913:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307561984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315704.525027:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307561984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 2103us (3166us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315704.525042:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1733 00010000:00010000:2.0:1761315704.539988:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315705.052474:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315705.052496:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058f03816 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322574208:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315705.052561:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315705.056254:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574208 00000100:00100000:3.0:1761315705.056328:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2933 00000100:00100000:3.0:1761315705.056334:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322574208:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315705.056399:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315705.056494:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322574208:12345-0@lo:41:osp-pre-1-0.0 Request processed in 159us (3933us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315705.056500:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2933 00000100:00100000:1.0:1761315705.057196:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058f03816 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322574208:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315705.563520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315706.587667:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315707.611608:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df02dd31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322574272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315707.611676:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315707.611748:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322574464:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315707.611754:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315707.611758:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315707.611761:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315707.611766:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3950 00000100:00080000:1.0:1761315707.611768:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315707.611772:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315707.611773:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315707.611790:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315707.611792:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c64d6e0e x1846869322574656/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315707.611793:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322574528:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315707.611804:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315707.611807:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315707.611808:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315707.611811:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3950 00000100:00080000:1.0:1761315707.611813:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315707.611813:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315707.611815:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315707.611823:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cf0de5dc x1846869322574720/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315707.611842:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574464 00000100:00100000:1.0:1761315707.611860:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322574912:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315707.611873:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2934 00000100:00100000:2.0:1761315707.611879:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322574464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315707.611885:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315707.611903:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315707.611907:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322574848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315707.611915:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322574464:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (142us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315707.611922:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2934 00000100:00100000:1.0:1761315707.611924:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315707.611943:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574912 00000100:00100000:2.0:1761315707.611960:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322574464:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315707.611964:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1534 00000100:00100000:1.0:1761315707.611968:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322574912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315707.611997:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322574912:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (110us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315707.612002:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1534 00000100:00100000:2.0:1761315707.612007:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322574912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612064:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574272 00000100:00100000:3.0:1761315707.612095:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1017 00000100:00100000:3.0:1761315707.612099:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322574272:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315707.612118:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315707.612170:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322574272:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (495us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612175:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1017 00000100:00100000:3.0:1761315707.612195:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322574336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612220:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315707.612237:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322574592:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612259:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315707.612274:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315707.612279:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322574400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612298:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315707.612309:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322574784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612327:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315707.612372:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574528 00000100:00100000:3.0:1761315707.612394:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574848 00000100:00100000:3.0:1761315707.612408:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574400 00000100:00100000:3.0:1761315707.612416:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574784 00000100:00100000:3.0:1761315707.612425:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4695 00000100:00100000:3.0:1761315707.612428:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006ee7382 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322574528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612457:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006ee7382 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322574528:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (644us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612463:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4695 00000100:00100000:3.0:1761315707.612472:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4696 00000100:00100000:3.0:1761315707.612475:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322574848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612498:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b358f44 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322574848:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (573us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612502:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4696 00000100:00100000:3.0:1761315707.612510:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4697 00000100:00100000:3.0:1761315707.612513:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000261cb5f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322574400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612537:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000261cb5f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322574400:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612541:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4697 00000100:00100000:3.0:1761315707.612549:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4698 00000100:00100000:3.0:1761315707.612553:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d59c57f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322574784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612575:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d59c57f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322574784:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612581:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4698 00000100:00100000:3.0:1761315707.612600:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574592 00000100:00100000:3.0:1761315707.612619:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2935 00000100:00100000:3.0:1761315707.612622:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001fa920b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322574592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612645:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001fa920b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322574592:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (386us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612650:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2935 00000100:00100000:3.0:1761315707.612669:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574336 00000100:00100000:3.0:1761315707.612688:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1734 00000100:00100000:3.0:1761315707.612691:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322574336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612715:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322574336:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (495us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315707.612720:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1734 00000100:00100000:3.0:1761315707.612763:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322574400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612774:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322574784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612781:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322574336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612788:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322574592:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.612814:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322574528:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315707.613000:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322574976:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315707.613024:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315707.613663:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574976 00000100:00100000:2.0:1761315707.613696:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2936 00000100:00100000:2.0:1761315707.613701:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322574976:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315707.613725:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315707.614215:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322574848:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315707.614231:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df02dd31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322574272:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315707.614674:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322574976:12345-0@lo:41:osp-pre-0-1.0 Request processed in 972us (1649us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315707.614682:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2936 00000100:00100000:2.0:1761315707.614703:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322574656:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315707.614736:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315707.614751:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf0de5dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322574720:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315707.614770:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315707.614812:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574656 00000100:00100000:3.0:1761315707.614909:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5d13e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322574976:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761315707.620987:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315707.621887:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322574720 00000100:00100000:1.0:1761315707.622693:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2309 00000100:00100000:1.0:1761315707.622701:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322574656:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315707.622814:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315707.622996:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2310 00000100:00100000:2.0:1761315707.623001:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322574720:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315707.623079:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315707.623808:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322574656:12345-0@lo:8:kworker/u8:0.0 Request processed in 1105us (9069us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315707.623818:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2309 00000100:00080000:3.0:1761315707.623910:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315707.623916:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315707.623921:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322574656:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315707.623937:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315707.623940:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315707.623943:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf0de5dc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322574720:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315707.623947:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322574720:12345-0@lo:8:kworker/u8:0.0 Request processed in 942us (9174us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315707.623956:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2310 00010000:00010000:0.0:1761315708.635640:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315709.640234:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315709.640287:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315709.640318:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315709.640428:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562432 00000100:00100000:0.0:1761315709.640457:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315709.640474:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2311 00000100:00100000:2.0:1761315709.640482:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307562432:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:3.0:1761315709.640501:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562240 00000100:00100000:3.0:1761315709.640533:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1735 00000100:00100000:3.0:1761315709.640538:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307562240:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00010000:02020000:2.0:1761315709.640576:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315709.640598:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307562240:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 58us (310us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315709.640607:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1735 00000100:00100000:2.0:1761315709.640614:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307562432:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 131us (386us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315709.640623:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2311 00000100:00100000:1.0:1761315709.641543:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562368 00000100:00100000:2.0:1761315709.641547:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562304 00000100:00100000:1.0:1761315709.641599:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4699 00000100:00100000:1.0:1761315709.641610:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000354757ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307562368:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315709.641610:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4700 00000100:00100000:2.0:1761315709.641619:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315709.641670:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000354757ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307562368:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (1351us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315709.641671:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086a5d13e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 52us (1216us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315709.641690:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4699 00000100:00100000:2.0:1761315709.641697:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4700 00010000:00010000:2.0:1761315709.660338:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315710.172246:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e403e677 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322575040:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315710.172322:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315710.173245:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575040 00000100:00100000:2.0:1761315710.173305:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2937 00000100:00100000:2.0:1761315710.173312:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322575040:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315710.173363:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315710.174527:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322575040:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1211us (2203us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315710.174537:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2937 00000100:00100000:0.0:1761315710.174623:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e403e677 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322575040:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315710.685050:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315711.707544:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315712.731743:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322575104:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315712.731752:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315712.731758:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761315712.731758:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322575168:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315712.731762:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315712.731769:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3955 00000100:00080000:0.0:1761315712.731772:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315712.731778:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315712.731799:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000572cc7ae x1846869322575424/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315712.731813:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315712.731816:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315712.731818:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315712.731821:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3955 00000100:00080000:0.0:1761315712.731823:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315712.731825:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315712.731845:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315712.731849:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095078c5a x1846869322575488/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315712.731881:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322575360:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.731907:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315712.731933:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315712.731938:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315712.731941:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea8b91f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322575552:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.731968:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315712.731974:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315712.731977:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322575744:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.732014:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024202054 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322575232:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732015:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315712.732023:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575104 00000100:00100000:2.0:1761315712.732051:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575168 00000100:00100000:0.0:1761315712.732055:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315712.732071:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1736 00000100:00100000:0.0:1761315712.732080:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322575616:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.732081:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322575104:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732085:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575552 00000100:00100000:2.0:1761315712.732122:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4701 00000100:00100000:0.0:1761315712.732127:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315712.732130:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322575168:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.732154:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322575104:12345-0@lo:400:kworker/u8:2.0 Request processed in 75us (309us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315712.732155:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761315712.732165:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322575296:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.732166:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1736 00000100:00100000:2.0:1761315712.732175:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a30fa38 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322575168:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (240us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315712.732183:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4701 00000100:00100000:0.0:1761315712.732200:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315712.732201:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575616 00000100:00100000:1.0:1761315712.732214:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575360 00000100:00100000:0.0:1761315712.732220:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322575680:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732220:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575296 00000100:00100000:2.0:1761315712.732242:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4702 00000100:00100000:1.0:1761315712.732246:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575232 00000100:00100000:2.0:1761315712.732246:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab5a3502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322575552:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.732252:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315712.732271:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2938 00000100:00100000:1.0:1761315712.732277:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322575360:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732284:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab5a3502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322575552:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315712.732290:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4702 00000100:00100000:2.0:1761315712.732308:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4703 00000100:00100000:0.0:1761315712.732316:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2939 00000100:00100000:2.0:1761315712.732316:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6b02a8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322575616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.732317:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322575360:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (408us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315712.732325:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322575232:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.732331:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2938 00000100:00100000:2.0:1761315712.732374:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6b02a8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322575616:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (249us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315712.732380:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4703 00000100:00100000:2.0:1761315712.732391:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4704 00000100:00100000:2.0:1761315712.732394:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d717e21f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322575296:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315712.732480:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322575424:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315712.732486:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322575232:12345-0@lo:400:kworker/u8:2.0 Request processed in 162us (433us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315712.732493:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2939 00000100:00100000:3.0:1761315712.732511:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315712.732526:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322575488:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315712.732543:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315712.732569:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d717e21f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322575296:12345-0@lo:400:kworker/u8:2.0 Request processed in 173us (371us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315712.732577:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4704 00010000:00010000:2.0:1761315712.732597:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315712.732601:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575744 00000100:00100000:2.0:1761315712.732618:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575680 00000100:00100000:0.0:1761315712.732625:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315712.732635:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575424 00000100:00100000:0.0:1761315712.732637:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322575872:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315712.732647:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575488 00000100:00100000:3.0:1761315712.732652:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1535 00000100:00100000:3.0:1761315712.732658:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322575744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732666:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1536 00000100:00100000:0.0:1761315712.732671:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315712.732675:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed3a4f84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322575680:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315712.732696:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322575744:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (683us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315712.732704:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1535 00000100:00100000:2.0:1761315712.732716:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed3a4f84 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322575680:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (467us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315712.732720:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2312 00000100:00100000:3.0:1761315712.732726:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc2cdad8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322575424:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315712.732729:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1536 00000100:00100000:2.0:1761315712.732745:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2313 00000100:00100000:2.0:1761315712.732748:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322575488:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315712.732847:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322575808:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315712.732875:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315712.733009:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322575744:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733020:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099fc3140 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322575168:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733051:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001883266f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322575296:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733059:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322575680:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733066:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024202054 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322575232:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733073:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322575616:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761315712.733110:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315712.733122:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575872 00010000:02020000:2.0:1761315712.733122:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315712.733149:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc2cdad8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322575424:12345-0@lo:8:kworker/u8:2.0 Request processed in 421us (638us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315712.733155:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2940 00000100:00100000:2.0:1761315712.733158:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322575488:12345-0@lo:8:kworker/u8:2.0 Request processed in 408us (613us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315712.733159:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2312 00000100:00100000:0.0:1761315712.733163:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322575872:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315712.733170:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2313 00000004:00000024:0.0:1761315712.733190:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315712.733196:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575808 00000100:00080000:3.0:1761315712.733198:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315712.733204:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315712.733208:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000572cc7ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322575424:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315712.733222:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315712.733225:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1018 00000100:00080000:3.0:1761315712.733227:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315712.733233:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322575808:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315712.733233:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322575488:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315712.733240:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322575872:12345-0@lo:41:osp-pre-0-1.0 Request processed in 76us (567us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315712.733249:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2940 00002000:00000024:2.0:1761315712.733256:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315712.733285:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322575872:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315712.733292:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322575808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (415us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315712.733299:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1018 00000100:00100000:3.0:1761315712.733313:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322575808:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315712.733489:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea8b91f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322575552:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.733502:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322575360:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315712.733510:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322575104:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315713.755751:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315714.761116:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315714.761160:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315714.761201:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315714.761291:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562624 00000100:00100000:1.0:1761315714.761376:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562560 00000100:00100000:1.0:1761315714.761397:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4705 00000100:00100000:1.0:1761315714.761402:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307562624:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315714.761410:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4706 00000100:00100000:0.0:1761315714.761415:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315714.761475:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fbcc5ee5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307562560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 59us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315714.761483:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4706 00000100:00100000:3.0:1761315714.761523:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562496 00000100:00100000:3.0:1761315714.761567:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1737 00000100:00100000:3.0:1761315714.761572:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307562496:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315714.762758:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aae74d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307562624:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1352us (1596us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315714.762769:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4705 00000100:00100000:3.0:1761315714.763043:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307562496:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1469us (1935us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315714.763051:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1737 00000100:00100000:1.0:1761315714.769062:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315714.769635:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562688 00000100:00100000:3.0:1761315714.770756:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2314 00000100:00100000:3.0:1761315714.770765:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000915b5df6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307562688:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315714.770857:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315714.770937:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000915b5df6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307562688:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 172us (1881us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315714.770945:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2314 00010000:00010000:0.0:1761315714.779560:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315715.291599:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315715.291622:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bdc06b77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322575936:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315715.291685:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315715.291778:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322575936 00000100:00100000:3.0:1761315715.291828:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2941 00000100:00100000:3.0:1761315715.291833:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322575936:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315715.291866:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315715.291920:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322575936:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315715.291927:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2941 00000100:00100000:1.0:1761315715.291968:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bdc06b77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322575936:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315715.806078:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315716.827569:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315717.851621:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc2cdad8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576000:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315717.851691:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315717.851770:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576000 00000100:00100000:2.0:1761315717.851827:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1019 00000100:00100000:2.0:1761315717.851834:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322576000:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315717.851867:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00010000:00010000:0.0:1761315717.851889:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315717.851912:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322576000:12345-0@lo:13:osp-pre-1-1.0 Request processed in 77us (221us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.851919:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1019 00000100:00100000:0.0:1761315717.851947:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc2cdad8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576000:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315717.852083:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315717.852092:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576064:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315717.852126:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315717.852137:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315717.852142:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315717.852145:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761315717.852150:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576256:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315717.852151:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3960 00000100:00080000:1.0:1761315717.852157:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315717.852161:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315717.852175:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315717.852182:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000daf1a052 x1846869322576448/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315717.852192:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1761315717.852198:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315717.852199:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322576320:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315717.852204:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315717.852206:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315717.852210:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3960 00000100:00080000:1.0:1761315717.852212:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315717.852215:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315717.852227:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315717.852229:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000054511f4f x1846869322576512/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315717.852256:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322576192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852283:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315717.852287:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322576128:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852299:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd36f033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322576576:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315717.852326:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315717.852332:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315717.852365:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000043ef4466 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322576384:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315717.852392:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315717.852395:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576064 00000100:00100000:2.0:1761315717.852426:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576256 00000100:00100000:1.0:1761315717.852435:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576384 00000100:00100000:2.0:1761315717.852440:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2942 00000100:00100000:2.0:1761315717.852444:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322576064:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315717.852461:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2943 00000004:00000024:2.0:1761315717.852467:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315717.852468:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322576256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852498:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322576064:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852504:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2942 00000100:00100000:1.0:1761315717.852515:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322576256:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (339us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852520:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2944 00000100:00100000:1.0:1761315717.852529:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2943 00000100:00100000:2.0:1761315717.852530:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063a505f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322576384:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852568:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063a505f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322576384:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (176us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852575:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2944 00000100:00100000:1.0:1761315717.852579:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576320 00000100:00100000:2.0:1761315717.852599:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576192 00000100:00100000:1.0:1761315717.852606:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576576 00000100:00100000:2.0:1761315717.852625:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4707 00000100:00100000:2.0:1761315717.852630:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322576320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315717.852634:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4708 00000100:00100000:1.0:1761315717.852640:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322576192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852668:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a347b1ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322576320:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (441us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852675:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4707 00000100:00100000:1.0:1761315717.852686:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049738da9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322576192:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (403us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852692:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4709 00000100:00100000:1.0:1761315717.852699:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4708 00000100:00100000:2.0:1761315717.852699:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322576576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852732:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008b165e1a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322576576:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.852738:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4709 00000100:00100000:1.0:1761315717.852743:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000043ef4466 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322576384:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852771:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322576192:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852782:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd36f033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322576576:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852802:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322576640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852825:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315717.852834:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322576448:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852861:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322576320:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852873:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576256:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315717.852873:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315717.852887:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322576064:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315717.852898:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322576512:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852907:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315717.852916:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322576704:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315717.852922:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315717.852947:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315717.852955:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576640 00000100:00100000:3.0:1761315717.852986:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4710 00000100:00100000:3.0:1761315717.852990:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322576640:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.852996:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576448 00000100:00100000:2.0:1761315717.853019:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576512 00000100:00100000:3.0:1761315717.853029:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013111502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322576640:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (204us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.853033:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576704 00000100:00100000:3.0:1761315717.853036:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4710 00000100:00100000:2.0:1761315717.853069:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1537 00000100:00100000:2.0:1761315717.853076:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322576704:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315717.853076:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2315 00000100:00100000:3.0:1761315717.853083:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322576448:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315717.853108:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322576704:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (163us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315717.853114:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1537 00000100:00100000:2.0:1761315717.853124:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2316 00000100:00100000:2.0:1761315717.853127:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322576512:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315717.853233:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315717.853266:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322576448:12345-0@lo:8:kworker/u8:0.0 Request processed in 184us (395us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315717.853272:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2315 00010000:02020000:2.0:1761315717.853278:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1761315717.853301:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315717.853305:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315717.853308:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322576448:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315717.853405:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576128 00000100:00100000:0.0:1761315717.853432:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1738 00000100:00100000:0.0:1761315717.853436:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322576128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315717.855249:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322576704:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.855262:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322576512:12345-0@lo:8:kworker/u8:0.0 Request processed in 2130us (2338us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315717.855271:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322576640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315717.855275:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2316 00000100:00080000:1.0:1761315717.855390:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315717.855395:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315717.855400:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322576128:12345-0@lo:400:kworker/u8:0.0 Request processed in 1961us (3074us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315717.855401:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322576512:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315717.855413:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1738 00000100:00100000:2.0:1761315717.855481:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322576128:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315718.877501:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315719.881315:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315719.881422:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315719.881462:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315719.881488:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315719.881517:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562752 00000100:00100000:1.0:1761315719.881559:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1739 00000100:00100000:1.0:1761315719.881565:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307562752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315719.881624:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307562752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 58us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315719.881631:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1739 00000100:00100000:1.0:1761315719.881679:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562816 00000100:00100000:1.0:1761315719.881704:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562880 00000100:00100000:1.0:1761315719.881726:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4711 00000100:00100000:1.0:1761315719.881731:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043ef4466 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307562816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315719.881760:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043ef4466 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307562816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 29us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315719.881777:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4711 00000100:00100000:1.0:1761315719.881788:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4712 00000100:00100000:1.0:1761315719.881792:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001fa5188a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307562880:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315719.881814:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001fa5188a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307562880:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 21us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315719.881819:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4712 00000100:00100000:2.0:1761315719.884213:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307562944 00000100:00100000:2.0:1761315719.884268:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2317 00000100:00100000:2.0:1761315719.884274:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307562944:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315719.884393:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315719.885585:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307562944:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 1307us (4273us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315719.885597:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2317 00010000:00010000:3.0:1761315719.899611:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315720.413121:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000238eabe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322576768:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315720.413187:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315720.413335:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576768 00000100:00100000:1.0:1761315720.413404:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2945 00000100:00100000:1.0:1761315720.413410:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322576768:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315720.413443:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315720.413495:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322576768:12345-0@lo:41:osp-pre-1-0.0 Request processed in 84us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315720.413502:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2945 00000100:00100000:2.0:1761315720.413699:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000238eabe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322576768:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315720.923531:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315721.948610:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315722.971617:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a3c465f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322576832:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315722.971687:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1761315722.971737:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315722.972181:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576832 00000100:00100000:3.0:1761315722.972239:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1020 00000100:00100000:3.0:1761315722.972245:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322576832:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315722.972276:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315722.972331:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322576832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 85us (645us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315722.972338:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1020 00000100:00100000:1.0:1761315722.972442:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a3c465f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322576832:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315722.973266:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087dbe404 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322576896:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315722.973317:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315722.973916:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ef2c853 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322576960:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315722.973948:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315722.973977:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315722.973982:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322577152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974004:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315722.974053:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577152 00000100:00100000:1.0:1761315722.974089:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4713 00000100:00100000:1.0:1761315722.974094:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008f353c47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322577152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974136:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008f353c47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322577152:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (131us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315722.974144:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4713 00000100:00100000:1.0:1761315722.974166:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576960 00000100:00100000:1.0:1761315722.974188:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2946 00000100:00100000:1.0:1761315722.974191:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322576960:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315722.974213:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315722.974239:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322576960:12345-0@lo:41:osp-pre-0-1.0 Request processed in 47us (291us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315722.974245:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2946 00000100:00100000:1.0:1761315722.974277:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322577152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974289:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ef2c853 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322576960:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315722.974331:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322577024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974374:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315722.974390:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315722.974394:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046eace05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322577088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974412:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315722.974432:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577088 00000100:00100000:1.0:1761315722.974452:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2947 00000100:00100000:1.0:1761315722.974456:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322577088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974503:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322577088:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (90us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315722.974509:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2947 00000100:00100000:1.0:1761315722.974525:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577024 00000100:00100000:1.0:1761315722.974543:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4714 00000100:00100000:1.0:1761315722.974547:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ef2c853 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322577024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974569:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ef2c853 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322577024:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (195us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315722.974573:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4714 00000100:00100000:1.0:1761315722.974591:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046eace05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322577088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.974598:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322577024:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315722.974834:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315722.974838:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315722.974841:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315722.974846:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3965 00000100:00080000:2.0:1761315722.974848:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315722.974852:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315722.974869:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000046eace05 x1846869322577280/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315722.974883:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315722.974886:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315722.974888:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315722.974890:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3965 00000100:00080000:2.0:1761315722.974892:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315722.974895:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315722.974902:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b91fac8e x1846869322577344/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315722.974948:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046eace05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322577280:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315722.974977:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315722.974992:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322577344:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975011:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315722.975049:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577280 00000100:00100000:2.0:1761315722.975066:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577344 00000100:00100000:2.0:1761315722.975078:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2318 00000100:00100000:2.0:1761315722.975082:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322577280:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315722.975171:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315722.975202:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322577280:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (225us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315722.975209:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2318 00000100:00100000:2.0:1761315722.975219:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2319 00000100:00100000:2.0:1761315722.975222:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322577344:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315722.975295:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315722.975318:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322577344:12345-0@lo:8:kworker/u8:2.0 Request processed in 95us (307us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315722.975323:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2319 00000100:00080000:2.0:1761315722.975366:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315722.975369:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315722.975372:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046eace05 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322577280:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315722.975379:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315722.975381:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315722.975382:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b91fac8e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322577344:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975397:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322577408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975416:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315722.975428:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315722.975433:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322577472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975448:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315722.975466:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577408 00000100:00100000:2.0:1761315722.975498:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577472 00000100:00100000:2.0:1761315722.975512:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4715 00000100:00100000:2.0:1761315722.975515:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322577408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975543:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322577408:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (127us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315722.975548:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4715 00000100:00100000:2.0:1761315722.975558:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4716 00000100:00100000:2.0:1761315722.975561:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322577472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975582:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322577472:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (133us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315722.975586:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4716 00000100:00100000:2.0:1761315722.975614:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322577472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315722.975622:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322577408:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315722.979291:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577536:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315722.979397:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315722.979455:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315722.979460:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577216:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315722.979486:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315722.979535:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577216 00000100:00100000:3.0:1761315722.979579:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2948 00000100:00100000:3.0:1761315722.979586:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009c0b045b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315722.979640:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009c0b045b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577216:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (153us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315722.979647:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2948 00000100:00100000:3.0:1761315722.979686:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a14c3afa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577216:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.982272:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577536 00000100:00100000:1.0:1761315722.982378:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1538 00000100:00100000:1.0:1761315722.982384:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315722.982464:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577536:12345-0@lo:400:kworker/u8:2.0 Request processed in 79us (3067us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315722.982471:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1538 00000100:00100000:3.0:1761315722.982703:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577536:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315722.982888:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322576896 00000100:00100000:0.0:1761315722.982921:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1740 00000100:00100000:0.0:1761315722.982925:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322576896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315722.982964:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322576896:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (9648us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315722.982970:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1740 00000100:00100000:3.0:1761315722.983046:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087dbe404 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322576896:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315723.995499:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315725.002488:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315725.002536:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315725.002573:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563072 00000100:00100000:1.0:1761315725.002579:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315725.002630:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4717 00000100:00100000:2.0:1761315725.002637:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307563072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315725.002651:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315725.002681:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563136 00000100:00100000:1.0:1761315725.002710:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4718 00000100:00100000:2.0:1761315725.002713:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb035f81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307563072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 73us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315725.002720:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315725.002727:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4717 00000100:00100000:1.0:1761315725.002755:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011072150 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315725.002762:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563008 00000100:00100000:1.0:1761315725.002762:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4718 00000100:00100000:0.0:1761315725.002796:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1741 00000100:00100000:0.0:1761315725.002803:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563008:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315725.002834:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563008:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (255us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315725.002840:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1741 00000100:00100000:3.0:1761315725.003150:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563200 00000100:00100000:3.0:1761315725.003184:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2320 00000100:00100000:3.0:1761315725.003188:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307563200:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315725.003284:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315725.005154:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307563200:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 1958us (2612us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315725.005170:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2320 00010000:00010000:3.0:1761315725.020016:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315725.531820:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315725.534169:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577600:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315725.534248:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315725.534364:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577600 00000100:00100000:1.0:1761315725.534705:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2949 00000100:00100000:1.0:1761315725.534710:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577600:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315725.534742:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315725.534789:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322577600:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (542us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315725.534795:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2949 00000100:00100000:2.0:1761315725.535007:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322577600:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315726.043587:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315727.067558:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315728.091479:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315728.091532:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007516f4b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322577728:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315728.091601:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315728.091633:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315728.091638:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315728.091641:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315728.091646:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3970 00000100:00080000:3.0:1761315728.091649:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315728.091655:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315728.091680:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc2cdad8 x1846869322578112/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315728.091696:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315728.091698:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315728.091700:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315728.091702:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3970 00000100:00080000:3.0:1761315728.091703:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315728.091705:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315728.091715:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bdc06b77 x1846869322578176/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315728.091758:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322577664:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315728.091787:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.091812:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004a3c465f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322577984:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.091833:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.091846:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058f03816 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322578368:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.091864:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.091883:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315728.091887:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322577920:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.091918:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577984 00000100:00100000:3.0:1761315728.091955:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4719 00000100:00100000:3.0:1761315728.091961:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322577984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092016:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c61786a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322577984:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (182us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315728.092023:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4719 00000100:00100000:3.0:1761315728.092045:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578368 00000100:00100000:3.0:1761315728.092064:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1539 00000100:00100000:3.0:1761315728.092068:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322578368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092094:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322578368:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (230us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315728.092100:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1539 00000100:00100000:3.0:1761315728.092135:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577664 00000100:00100000:3.0:1761315728.092155:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1021 00000100:00100000:3.0:1761315728.092158:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322577664:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315728.092176:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315728.092204:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322577664:12345-0@lo:13:osp-pre-1-1.0 Request processed in 44us (416us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315728.092209:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1021 00000100:00100000:3.0:1761315728.092224:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322577856:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092246:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.092257:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e403e677 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322578240:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092274:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.092286:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315728.092291:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322577792:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092311:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315728.092320:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000238eabe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322578048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.092380:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577856 00000100:00100000:3.0:1761315728.092399:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578240 00000100:00100000:3.0:1761315728.092414:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4720 00000100:00100000:2.0:1761315728.092653:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc2cdad8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322578112:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315728.092683:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315728.092705:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bdc06b77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322578176:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315728.092728:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315728.092753:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578112 00000100:00100000:2.0:1761315728.092769:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578176 00000100:00100000:2.0:1761315728.092780:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2321 00000100:00100000:2.0:1761315728.092784:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322578112:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315728.092865:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315728.092908:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322578112:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (221us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315728.092914:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2321 00000100:00100000:2.0:1761315728.092924:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2322 00000100:00100000:2.0:1761315728.092927:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322578176:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315728.093008:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315728.093177:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315728.093315:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577728 00000100:00100000:1.0:1761315728.093365:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2950 00000100:00100000:1.0:1761315728.093369:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322577728:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315728.093393:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315728.093434:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322577728:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (1834us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315728.093440:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2950 00000100:00100000:1.0:1761315728.093502:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007516f4b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322577728:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315728.093552:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4721 00000100:00100000:1.0:1761315728.093557:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038a7bd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322578240:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315728.093589:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038a7bd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322578240:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (1314us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315728.093594:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4721 00000100:00100000:1.0:1761315728.093621:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577792 00000100:00100000:1.0:1761315728.093644:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1742 00000100:00100000:1.0:1761315728.093647:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322577792:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315728.093690:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322577792:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (1378us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315728.093696:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1742 00000100:00080000:1.0:1761315728.093786:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315728.093791:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315728.093795:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc2cdad8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322578112:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315728.093950:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322578176:12345-0@lo:8:kworker/u8:2.0 Request processed in 1020us (1220us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315728.093956:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2322 00000100:00100000:0.0:1761315728.094199:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322578432:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094228:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315728.094249:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e403e677 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322578240:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094266:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322577792:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094285:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315728.094302:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df02dd31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322578304:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094324:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315728.094361:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006766d617 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322577664:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315728.094374:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004a3c465f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322577984:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094382:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058f03816 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322578368:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094408:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578304 00000100:00100000:0.0:1761315728.094432:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4722 00000100:00100000:0.0:1761315728.094437:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e178d65c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322578304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094470:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e178d65c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322578304:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (145us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315728.094476:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4722 00000100:00100000:0.0:1761315728.094501:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df02dd31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322578304:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094516:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578048 00000100:00100000:0.0:1761315728.094533:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322577920 00000100:00100000:0.0:1761315728.094548:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2951 00000100:00100000:0.0:1761315728.094551:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322578048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094578:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322578048:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (1404us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315728.094584:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2951 00000100:00100000:0.0:1761315728.094595:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2952 00000100:00100000:0.0:1761315728.094598:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322577920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094624:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322577920:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315728.094629:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2952 00000100:00100000:0.0:1761315728.094654:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322577920:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315728.094674:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000238eabe1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322578048:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315728.094807:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578432 00000100:00100000:2.0:1761315728.094848:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1540 00000100:00100000:2.0:1761315728.094852:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322578432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315728.094888:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322578432:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (660us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315728.094894:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1540 00000100:00100000:2.0:1761315728.094917:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322578432:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315728.095191:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315728.095196:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315728.095199:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bdc06b77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322578176:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315728.095656:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322577856:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315728.097340:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b7d9eb5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322577856:12345-0@lo:400:kworker/u8:2.0 Request processed in 4915us (5088us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315728.097368:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4720 00000100:00100000:2.0:1761315728.097699:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f86fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322577856:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315729.116707:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315730.120698:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315730.120775:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563328 00000100:00100000:0.0:1761315730.120835:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4723 00000100:00100000:0.0:1761315730.120842:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307563328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315730.120922:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007516f4b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307563328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 80us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315730.120930:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4723 00000100:00100000:2.0:1761315730.121130:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315730.121168:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315730.121181:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563264 00000100:00100000:1.0:1761315730.121213:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1743 00000100:00100000:1.0:1761315730.121219:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315730.121269:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 50us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315730.121276:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1743 00000100:00100000:3.0:1761315730.121368:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563392 00000100:00100000:3.0:1761315730.121401:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4724 00000100:00100000:3.0:1761315730.121405:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a254e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315730.121433:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a254e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 27us (267us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315730.121439:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4724 00000100:00100000:2.0:1761315730.123288:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315730.123387:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563456 00000100:00100000:0.0:1761315730.123429:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2323 00000100:00100000:0.0:1761315730.123434:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307563456:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315730.123527:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315730.123593:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307563456:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 157us (310us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315730.123600:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2323 00010000:00010000:0.0:1761315730.139573:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315730.651781:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315730.651796:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322578496:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315730.651860:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315730.651953:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578496 00000100:00100000:0.0:1761315730.652040:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2953 00000100:00100000:0.0:1761315730.652045:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322578496:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315730.652078:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315730.652139:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322578496:12345-0@lo:41:osp-pre-1-0.0 Request processed in 93us (280us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315730.652146:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2953 00000100:00100000:1.0:1761315730.652292:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322578496:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315731.164514:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315732.190384:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315733.211591:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315733.211654:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322578560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315733.211722:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315733.211832:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578560 00000100:00100000:0.0:1761315733.211884:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1022 00000100:00100000:0.0:1761315733.211891:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322578560:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315733.211918:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315733.212796:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322578624:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315733.212916:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322578560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1025us (1194us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315733.212925:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1022 00000100:00100000:0.0:1761315733.213043:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322578688:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315733.213076:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315733.213138:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578688 00000100:00100000:0.0:1761315733.213167:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2954 00000100:00100000:0.0:1761315733.213171:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b3e1df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322578688:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315733.213195:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315733.213227:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b3e1df5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322578688:12345-0@lo:41:osp-pre-0-1.0 Request processed in 55us (152us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315733.213233:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2954 00000100:00100000:0.0:1761315733.213575:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033ca1fdd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322578688:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761315733.214461:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315733.214463:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322578560:0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761315733.214467:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315733.214471:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315733.214478:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3975 00000100:00080000:1.0:1761315733.214481:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315733.214486:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315733.214493:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322578816:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.214507:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000444a05f3 x1846869322578944/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315733.214526:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315733.214529:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315733.214530:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315733.214534:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3975 00000100:00080000:1.0:1761315733.214537:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315733.214538:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315733.214542:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315733.214554:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001d4f7a52 x1846869322579008/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315733.214565:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315733.214573:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322578880:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315733.214599:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.214605:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322578944:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315733.214636:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322579200:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.214642:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.214665:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322579008:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315733.214671:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.214697:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315733.214713:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578880 00000100:00100000:1.0:1761315733.214734:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578816 00000100:00100000:3.0:1761315733.214752:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2955 00000100:00100000:3.0:1761315733.214758:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322578880:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.214768:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4725 00000100:00100000:1.0:1761315733.214773:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008c6ae23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322578816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315733.214807:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322578880:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315733.214815:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008c6ae23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322578816:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315733.214821:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2955 00000100:00100000:1.0:1761315733.214827:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4725 00000100:00100000:3.0:1761315733.214849:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578944 00000100:00100000:1.0:1761315733.214854:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579200 00000100:00100000:3.0:1761315733.214873:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579008 00000100:00100000:1.0:1761315733.214887:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1541 00000100:00100000:1.0:1761315733.214893:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322579200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315733.214898:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2324 00000100:00100000:3.0:1761315733.214903:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd36f033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322578944:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315733.214923:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322579200:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315733.214929:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1541 00000100:00100000:1.0:1761315733.214940:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2325 00000100:00100000:1.0:1761315733.214953:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322579008:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315733.215106:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315733.215139:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd36f033 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322578944:12345-0@lo:8:kworker/u8:0.0 Request processed in 235us (497us total) trans 0 rc -19/-19 00010000:02020000:1.0:1761315733.215143:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315733.215149:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2324 00000100:00100000:1.0:1761315733.215174:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322579008:12345-0@lo:8:kworker/u8:0.0 Request processed in 229us (478us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315733.215181:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2325 00000100:00100000:3.0:1761315733.215190:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322579200:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315733.215203:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322578880:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315733.215216:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322578816:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315733.215222:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315733.215227:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315733.215231:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322578944:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315733.215241:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315733.215244:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315733.215246:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322579008:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315733.215271:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322579072:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.215306:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.215337:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315733.215358:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.215367:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322578752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.215393:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315733.215410:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579072 00000100:00100000:1.0:1761315733.215411:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322579136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.215435:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315733.215463:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578624 00000100:00100000:1.0:1761315733.215488:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1744 00000100:00100000:1.0:1761315733.215492:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322578624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315733.215525:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322578624:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (183us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315733.215534:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1744 00000100:00100000:2.0:1761315733.216165:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322578752 00000100:00100000:2.0:1761315733.216203:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579136 00000100:00100000:2.0:1761315733.216217:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4726 00000100:00100000:2.0:1761315733.216221:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000012cee7c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322579072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216265:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000012cee7c9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322579072:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (958us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315733.216271:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4726 00000100:00100000:2.0:1761315733.216284:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4727 00000100:00100000:2.0:1761315733.216287:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a986d022 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322578752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216311:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a986d022 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322578752:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (919us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315733.216316:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4727 00000100:00100000:2.0:1761315733.216327:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4728 00000100:00100000:2.0:1761315733.216330:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001fa920b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322579136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216379:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001fa920b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322579136:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (944us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315733.216384:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4728 00000100:00100000:2.0:1761315733.216414:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322578752:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216424:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322579136:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216432:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322579072:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315733.216464:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322578624:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315734.236036:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315735.247855:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315735.249434:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315735.250010:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563520 00000100:00100000:0.0:1761315735.250059:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1745 00000100:00100000:0.0:1761315735.250066:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315735.250131:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307563520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 64us (2285us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315735.250139:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1745 00000100:00100000:2.0:1761315735.250491:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315735.250520:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315735.251118:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563648 00000100:00100000:3.0:1761315735.251158:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563584 00000100:00100000:3.0:1761315735.251180:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4729 00000100:00100000:3.0:1761315735.251184:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315735.251229:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 44us (1799us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315735.251236:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4729 00000100:00100000:3.0:1761315735.251248:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4730 00000100:00100000:3.0:1761315735.251252:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307563584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315735.251274:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307563584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (785us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315735.251283:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4730 00000100:00100000:0.0:1761315735.251824:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563712 00000100:00100000:0.0:1761315735.251857:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2326 00000100:00100000:0.0:1761315735.251862:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307563712:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315735.251965:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315735.252006:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307563712:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 144us (1485us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315735.252013:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2326 00010000:00010000:3.0:1761315735.261822:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315735.773052:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315735.773084:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322579264:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315735.773176:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315735.773390:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579264 00000100:00100000:3.0:1761315735.773457:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2956 00000100:00100000:3.0:1761315735.773464:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322579264:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315735.773509:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315735.773581:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322579264:12345-0@lo:41:osp-pre-1-0.0 Request processed in 115us (406us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315735.773588:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2956 00000100:00100000:1.0:1761315735.773698:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322579264:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315736.283740:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315737.307690:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315738.331643:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:0.0:1761315738.331675:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315738.331679:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322579328:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315738.331779:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315738.332360:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315738.332367:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761315738.332372:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322579456:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315738.332372:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315738.332379:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3980 00000100:00080000:3.0:1761315738.332384:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315738.332390:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315738.332417:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bca5f7bb x1846869322579648/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315738.332420:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315738.332439:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315738.332443:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315738.332445:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761315738.332447:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1761315738.332449:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3980 00000100:00080000:3.0:1761315738.332453:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:1.0:1761315738.332455:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322579520:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315738.332458:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315738.332476:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a5918643 x1846869322579712/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315738.332490:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.332531:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322579840:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315738.332540:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322579648:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315738.332556:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315738.332798:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322579904:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315738.333474:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579328 00000100:00100000:0.0:1761315738.333538:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2957 00000100:00100000:0.0:1761315738.333546:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322579328:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315738.333590:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315738.333720:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315738.333750:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322579712:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315738.333782:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315738.333821:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579456 00000100:00100000:3.0:1761315738.333848:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579520 00000100:00100000:3.0:1761315738.333865:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579840 00000100:00100000:3.0:1761315738.333884:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4731 00000100:00100000:3.0:1761315738.333888:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009c0b045b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322579456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315738.333938:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009c0b045b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322579456:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (1520us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315738.333945:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4731 00000100:00100000:3.0:1761315738.333963:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4732 00000100:00100000:3.0:1761315738.333966:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322579520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315738.333993:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f86fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322579520:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1507us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315738.333999:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4732 00000100:00100000:3.0:1761315738.334008:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4733 00000100:00100000:3.0:1761315738.334011:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bdc06b77 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322579840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315738.334038:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bdc06b77 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322579840:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1484us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315738.334044:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4733 00000100:00100000:3.0:1761315738.334065:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579648 00000100:00100000:3.0:1761315738.334085:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579712 00000100:00100000:3.0:1761315738.334101:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2327 00000100:00100000:3.0:1761315738.334105:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322579648:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334263:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322579840:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334278:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322579520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334290:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322579456:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334337:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322579392:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334395:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.334412:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322579776:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334429:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.334440:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322580032:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315738.334460:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.334475:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315738.334480:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322579584:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334496:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.334508:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322579968:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334522:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315738.334543:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579776 00000100:00100000:1.0:1761315738.334565:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4734 00000100:00100000:1.0:1761315738.334568:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000238eabe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322579776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334596:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000238eabe1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322579776:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (166us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315738.334601:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4734 00000100:00100000:1.0:1761315738.334624:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322579776:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334673:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579392 00000100:00100000:1.0:1761315738.334695:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1746 00000100:00100000:1.0:1761315738.334699:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322579392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334733:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322579392:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (338us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315738.334738:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1746 00000100:00100000:1.0:1761315738.334766:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579584 00000100:00100000:1.0:1761315738.334785:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2958 00000100:00100000:1.0:1761315738.334789:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df02dd31 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322579584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334818:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df02dd31 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322579584:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (320us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315738.334823:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2958 00000100:00100000:1.0:1761315738.334846:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322579584:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334854:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322579392:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334875:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579968 00000100:00100000:1.0:1761315738.334899:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1542 00000100:00100000:1.0:1761315738.334903:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322579968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.334954:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322579968:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (431us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315738.334960:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1542 00000100:00100000:1.0:1761315738.334971:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2328 00000100:00100000:1.0:1761315738.334974:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322579712:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315738.335072:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315738.335108:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322579712:12345-0@lo:8:kworker/u8:2.0 Request processed in 132us (1325us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315738.335114:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2328 00000100:00080000:1.0:1761315738.335147:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315738.335152:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315738.335156:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322579712:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315738.335179:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322579968:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315738.335214:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580032 00000100:00100000:1.0:1761315738.335241:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1761315738.335246:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322580032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315738.335269:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315738.335304:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322580032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (843us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315738.335311:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1023 00000100:00100000:1.0:1761315738.335369:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322580032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315738.335592:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315738.335720:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322579328:12345-0@lo:41:osp-pre-0-1.0 Request processed in 2173us (3942us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315738.335728:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2957 00000100:00100000:2.0:1761315738.335814:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322579328:0@lo:41:osp-pre-0-1.0 00010000:02020000:3.0:1761315738.335974:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315738.336019:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322579648:12345-0@lo:8:kworker/u8:2.0 Request processed in 1913us (2300us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315738.336026:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2327 00000100:00100000:3.0:1761315738.336040:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322579904 00000100:00100000:3.0:1761315738.336058:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1543 00000100:00100000:3.0:1761315738.336061:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322579904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315738.336086:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322579904:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (496us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315738.336091:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1543 00000100:00080000:0.0:1761315738.336164:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315738.336168:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315738.336172:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322579648:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315738.336228:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322579904:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315739.359976:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315740.360058:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315740.360854:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315740.360965:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563776 00000100:00100000:1.0:1761315740.361025:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1747 00000100:00100000:1.0:1761315740.361031:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307563776:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315740.361106:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307563776:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 73us (1054us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315740.361115:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1747 00000100:00100000:3.0:1761315740.364017:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563840 00000100:00100000:2.0:1761315740.364059:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315740.364921:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563904 00000100:00100000:1.0:1761315740.364979:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4735 00000100:00100000:1.0:1761315740.364985:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052a02902 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307563840:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315740.365052:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052a02902 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307563840:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 66us (4200us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315740.365063:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4736 00000100:00100000:1.0:1761315740.365073:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4735 00000100:00100000:3.0:1761315740.365079:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563904:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315740.365148:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307563904:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (1097us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315740.365156:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4736 00000100:00100000:3.0:1761315740.365233:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315740.366675:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307563968 00000100:00100000:1.0:1761315740.366725:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2329 00000100:00100000:1.0:1761315740.366729:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307563968:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315740.366842:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315740.366906:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307563968:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 176us (1673us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315740.366913:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2329 00010000:00010000:1.0:1761315740.382842:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315740.891580:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322580096:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315740.891654:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315740.892123:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580096 00000100:00100000:1.0:1761315740.893406:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2959 00000100:00100000:1.0:1761315740.893419:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322580096:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315740.893478:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315740.897473:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322580096:12345-0@lo:41:osp-pre-1-0.0 Request processed in 4040us (5805us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315740.897497:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2959 00000100:00100000:2.0:1761315740.897649:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322580096:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315741.404464:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315742.427849:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315743.451667:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322580160:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315743.451770:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315743.452198:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580160 00000100:00100000:0.0:1761315743.452264:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2960 00000100:00100000:0.0:1761315743.452272:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322580160:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315743.452315:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315743.452396:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322580160:12345-0@lo:41:osp-pre-0-1.0 Request processed in 123us (628us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315743.452403:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2960 00000100:00100000:2.0:1761315743.452498:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322580160:0@lo:41:osp-pre-0-1.0 00010000:00010000:1.0:1761315743.452724:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315743.452850:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322580224:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315743.452968:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.453828:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322580288:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315743.453841:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315743.453849:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315743.453853:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315743.453862:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3985 00000100:00080000:3.0:1761315743.453865:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315743.453883:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315743.453915:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e6276fbf x1846869322580544/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315743.453921:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315743.453945:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315743.453950:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315743.453953:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315743.453959:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3985 00000100:00080000:3.0:1761315743.453961:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315743.453964:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315743.453970:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315743.453980:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000072c6c69d x1846869322580608/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315743.453981:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580480:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454040:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.454075:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b6424f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322580672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454112:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.454132:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315743.454138:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580864:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454167:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.454206:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580288 00000100:00100000:2.0:1761315743.454257:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1748 00000100:00100000:2.0:1761315743.454263:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322580288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454314:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322580288:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (403us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315743.454322:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1748 00000100:00100000:2.0:1761315743.454388:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580672 00000100:00100000:2.0:1761315743.454415:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4737 00000100:00100000:2.0:1761315743.454420:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322580672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454462:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003952b204 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322580672:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315743.454469:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4737 00000100:00100000:2.0:1761315743.454505:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580864 00000100:00100000:3.0:1761315743.454519:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322580352:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.454538:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1544 00000100:00100000:2.0:1761315743.454544:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315743.455697:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580224 00000100:00100000:1.0:1761315743.455753:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1024 00000100:00100000:1.0:1761315743.455759:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322580224:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315743.455792:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315743.456074:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580864:12345-0@lo:400:kworker/u8:2.0 Request processed in 1527us (1905us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315743.456085:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1544 00000100:00100000:3.0:1761315743.456113:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.456161:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580864:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456175:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322580736:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.456183:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b6424f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322580672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.456199:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000019063863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322580288:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456227:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315743.456270:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315743.456276:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580352 00000100:00100000:3.0:1761315743.456282:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322580416:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456314:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315743.456337:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322580800:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456393:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.456424:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580736 00000100:00100000:3.0:1761315743.456454:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580416 00000100:00100000:2.0:1761315743.456479:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4738 00000100:00100000:2.0:1761315743.456491:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000beffc925 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322580352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456508:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4739 00000100:00100000:0.0:1761315743.456516:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322580544:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315743.456520:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d131fbc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322580736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.456563:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000beffc925 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322580352:12345-0@lo:400:kworker/u8:2.0 Request processed in 74us (455us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315743.456583:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315743.456585:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4738 00000100:00100000:1.0:1761315743.456591:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322580224:12345-0@lo:13:osp-pre-1-1.0 Request processed in 829us (3622us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315743.456602:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1024 00000100:00100000:0.0:1761315743.456613:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322580608:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315743.456616:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4740 00000100:00100000:2.0:1761315743.456626:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba8be421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322580416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315743.456640:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580800 00000100:00100000:0.0:1761315743.456646:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315743.456675:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580544 00000100:00100000:2.0:1761315743.456685:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba8be421 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322580416:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (369us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315743.456696:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4740 00000100:00100000:0.0:1761315743.456707:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580608 00000100:00100000:1.0:1761315743.456713:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1545 00000100:00100000:3.0:1761315743.456713:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d131fbc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322580736:12345-0@lo:400:kworker/u8:2.0 Request processed in 185us (477us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315743.456723:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322580800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315743.456741:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2330 00000100:00100000:3.0:1761315743.456749:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4739 00000100:00100000:0.0:1761315743.456756:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2331 00000100:00100000:2.0:1761315743.456756:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322580544:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315743.456768:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322580608:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315743.456787:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322580800:12345-0@lo:400:kworker/u8:2.0 Request processed in 60us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315743.456795:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1545 00000100:00100000:3.0:1761315743.457022:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000daf1a052 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322580416:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315743.457034:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322580224:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315743.457046:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322580800:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.457070:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322580352:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315743.457081:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322580736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315743.457109:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580480 00000100:00100000:1.0:1761315743.457157:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2961 00000100:00100000:1.0:1761315743.457164:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315743.457219:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580480:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (3182us total) trans 0 rc 0/0 00010000:02020000:2.0:1761315743.457225:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315743.457228:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2961 00010000:02020000:0.0:1761315743.457240:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315743.457305:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322580608:12345-0@lo:8:kworker/u8:2.0 Request processed in 539us (660us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315743.457311:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000063a505f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580480:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315743.457319:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2331 00000100:00100000:2.0:1761315743.457327:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322580544:12345-0@lo:8:kworker/u8:2.0 Request processed in 564us (745us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315743.457338:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2330 00000100:00080000:1.0:1761315743.457380:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315743.457387:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315743.457394:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6276fbf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322580544:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315743.457411:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315743.457415:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315743.457418:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322580608:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315744.475553:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315745.481555:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315745.482159:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315745.482261:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315745.482265:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564096 00000100:00100000:2.0:1761315745.482311:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315745.482333:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564160 00000100:00100000:2.0:1761315745.482339:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564224 00000100:00100000:3.0:1761315745.482383:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4741 00000100:00100000:2.0:1761315745.482392:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2332 00000100:00100000:3.0:1761315745.482393:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307564096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315745.482396:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307564224:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:0.0:1761315745.482404:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564032 00000100:00100000:1.0:1761315745.482442:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4742 00000100:00100000:0.0:1761315745.482447:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1749 00000100:00100000:1.0:1761315745.482453:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307564160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315745.482455:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307564032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315745.482484:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000059c6fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307564096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 92us (326us total) trans 0 rc 0/0 00010000:02020000:2.0:1761315745.482490:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315745.482493:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4741 00000100:00100000:0.0:1761315745.482500:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307564032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 45us (955us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315745.482514:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1749 00000100:00100000:1.0:1761315745.482521:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001345af7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307564160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 68us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315745.482531:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4742 00000100:00100000:2.0:1761315745.482536:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307564224:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 137us (223us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315745.482544:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2332 00010000:00010000:3.0:1761315745.501098:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315746.011819:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580928:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315746.011949:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315746.016381:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580928 00000100:00100000:1.0:1761315746.016480:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2962 00000100:00100000:1.0:1761315746.016488:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580928:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315746.016542:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315746.016643:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322580928:12345-0@lo:41:osp-pre-1-0.0 Request processed in 155us (4695us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315746.016651:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2962 00000100:00100000:3.0:1761315746.016847:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322580928:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315746.524080:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315747.547592:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315748.571571:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322580992:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315748.571589:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322581056:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.571594:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322581120:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315748.571608:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315748.571615:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315748.571621:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315748.571628:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3991 00000100:00080000:3.0:1761315748.571631:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315748.571654:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315748.571679:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315748.571679:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.571693:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1a5e34c x1846869322581440/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315748.571716:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315748.571719:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d75a9b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581248:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315748.571722:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315748.571725:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315748.571729:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3991 00000100:00080000:3.0:1761315748.571732:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315748.571735:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315748.571749:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.571751:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d029cd1d x1846869322581504/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315748.571771:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315748.571780:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322581312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.571813:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.571813:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322581184:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.571847:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322581632:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.571867:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.571896:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322581568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.571899:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581056 00000100:00100000:3.0:1761315748.571924:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315748.571931:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315748.571960:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1750 00000100:00100000:0.0:1761315748.571969:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067d9ad19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322581376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.571970:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322581056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.571984:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581312 00000100:00100000:0.0:1761315748.572004:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.572024:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581184 00000100:00100000:0.0:1761315748.572035:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322581760:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.572039:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322581056:12345-0@lo:400:kworker/u8:2.0 Request processed in 68us (359us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315748.572047:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581568 00000100:00100000:2.0:1761315748.572051:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1750 00000100:00100000:3.0:1761315748.572073:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4743 00000100:00100000:3.0:1761315748.572078:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322581312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572085:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315748.572099:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581248 00000100:00100000:2.0:1761315748.572136:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581120 00000100:00100000:3.0:1761315748.572141:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6276fbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322581312:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (329us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315748.572143:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581376 00000100:00100000:3.0:1761315748.572152:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4743 00000100:00100000:2.0:1761315748.572153:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2963 00000100:00100000:2.0:1761315748.572160:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322581248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572178:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2964 00000100:00100000:3.0:1761315748.572178:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4744 00000100:00100000:3.0:1761315748.572189:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322581184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572190:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322581120:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315748.572200:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322581248:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (451us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315748.572207:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2963 00000100:00100000:2.0:1761315748.572226:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2965 00000004:00000024:0.0:1761315748.572227:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315748.572232:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322581184:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (364us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315748.572233:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322581376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.572244:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4744 00000100:00100000:3.0:1761315748.572259:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4745 00000100:00100000:3.0:1761315748.572264:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322581568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572284:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322581120:12345-0@lo:41:osp-pre-0-1.0 Request processed in 96us (354us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315748.572286:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322581376:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (282us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315748.572296:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322581568:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (372us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315748.572300:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2964 00000100:00100000:2.0:1761315748.572309:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2965 00000100:00100000:3.0:1761315748.572309:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4745 00000100:00100000:0.0:1761315748.572333:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581760 00000100:00100000:2.0:1761315748.572384:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.572394:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322581184:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572397:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1546 00000100:00100000:0.0:1761315748.572405:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322581760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.572417:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322581568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.572418:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322581312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.572439:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d75a9b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581248:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572450:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322581760:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315748.572456:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee723cf2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322581056:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.572469:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581632 00000100:00100000:0.0:1761315748.572474:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1546 00000100:00100000:2.0:1761315748.572477:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315748.572486:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581696:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.572496:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4746 00000100:00100000:3.0:1761315748.572502:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000063a505f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322581632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315748.572514:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315748.572531:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067d9ad19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322581376:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315748.572542:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000063a505f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322581632:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (157us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315748.572548:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4746 00000100:00100000:2.0:1761315748.572553:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322581632:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572554:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322581760:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572580:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322581120:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315748.572611:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581696 00000100:00100000:0.0:1761315748.572633:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1547 00000100:00100000:0.0:1761315748.572637:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322581696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315748.572676:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322581696:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315748.572682:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1547 00010000:00010000:0.0:1761315748.572896:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315748.573194:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322580992 00000100:00100000:0.0:1761315748.573227:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1025 00000100:00100000:0.0:1761315748.573231:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322580992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315748.573255:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315748.573288:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322580992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (1611us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315748.573294:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1025 00000100:00100000:2.0:1761315748.574649:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581696:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315748.574898:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a3d67ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322580992:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315748.577606:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1a5e34c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322581440:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315748.577723:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315748.577780:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322581504:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315748.577805:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315748.577912:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581440 00000100:00100000:1.0:1761315748.577979:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581504 00000100:00100000:1.0:1761315748.578009:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2333 00000100:00100000:1.0:1761315748.578015:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322581440:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315748.578060:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2334 00000100:00100000:3.0:1761315748.578065:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322581504:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315748.578228:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761315748.578262:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315748.578295:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322581440:12345-0@lo:8:kworker/u8:2.0 Request processed in 278us (573us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315748.578303:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322581504:12345-0@lo:8:kworker/u8:2.0 Request processed in 235us (496us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315748.578305:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2333 00000100:00100000:3.0:1761315748.578314:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2334 00000100:00080000:3.0:1761315748.578388:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315748.578394:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315748.578398:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1a5e34c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322581440:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315748.578406:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315748.578409:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315748.578411:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322581504:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315749.595627:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315750.604116:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315750.604188:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315750.604226:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315750.604282:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315750.604338:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564416 00000100:00100000:2.0:1761315750.604449:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564352 00000100:00100000:1.0:1761315750.604462:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4747 00000100:00100000:1.0:1761315750.604472:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307564416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315750.604568:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000019063863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307564416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 95us (341us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315750.605775:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564288 00000100:00100000:0.0:1761315750.605832:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1751 00000100:00100000:0.0:1761315750.605837:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307564288:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315750.605887:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307564288:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (1785us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315750.605894:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1751 00000100:00100000:3.0:1761315750.606551:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564480 00000100:00100000:3.0:1761315750.606579:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2335 00000100:00100000:3.0:1761315750.606583:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000daf1a052 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307564480:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315750.606750:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315750.606814:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000daf1a052 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307564480:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 228us (2625us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315750.606830:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2335 00000100:00100000:1.0:1761315750.607556:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4747 00000100:00100000:1.0:1761315750.607591:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4748 00000100:00100000:1.0:1761315750.607599:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2000c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307564352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315750.607653:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2000c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307564352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (3372us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315750.607660:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4748 00010000:00010000:3.0:1761315750.620412:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315751.131658:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322581824:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315751.131762:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315751.131922:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581824 00000100:00100000:1.0:1761315751.132090:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2966 00000100:00100000:1.0:1761315751.132099:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322581824:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315751.132149:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315751.132219:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322581824:12345-0@lo:41:osp-pre-1-0.0 Request processed in 120us (459us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315751.132227:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2966 00000100:00100000:0.0:1761315751.132725:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322581824:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315751.643947:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315752.667573:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315753.692408:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f32569ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581888:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315753.692487:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315753.692567:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581888 00000100:00100000:0.0:1761315753.692969:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322581952:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315753.693003:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315753.693041:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322581952 00000100:00100000:0.0:1761315753.693092:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2967 00000100:00100000:0.0:1761315753.693098:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322581952:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315753.693128:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315753.693163:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322581952:12345-0@lo:41:osp-pre-0-1.0 Request processed in 65us (160us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315753.693170:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2967 00000100:00100000:0.0:1761315753.693213:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001ca88bcf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322581952:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315753.693220:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1026 00000100:00100000:3.0:1761315753.693230:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322581888:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315753.693272:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00010000:00010000:0.0:1761315753.693308:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315753.693393:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322581888:12345-0@lo:13:osp-pre-1-1.0 Request processed in 163us (906us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315753.693400:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1026 00000100:00080000:2.0:1761315753.693900:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315753.693908:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008250083e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322582016:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315753.693908:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315753.693913:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315753.693922:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 3996 00000100:00080000:2.0:1761315753.693926:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315753.693930:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315753.693959:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cf3a4110 x1846869322582272/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315753.693972:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315753.693975:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315753.693977:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315753.693980:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 3996 00000100:00080000:2.0:1761315753.693982:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315753.693984:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315753.693994:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e2d157aa x1846869322582336/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315753.694051:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047e6259e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322582208:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.694970:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f32569ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322581888:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315753.695002:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322582144:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695050:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315753.695078:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322582272:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695080:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000065048840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322582528:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315753.695125:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315753.695144:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322582336:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315753.695166:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315753.695203:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582272 00000100:00100000:3.0:1761315753.695236:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582336 00000100:00100000:3.0:1761315753.695256:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2336 00000100:00100000:1.0:1761315753.695271:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315753.695325:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322582400:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695395:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315753.695403:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315753.695460:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315753.695470:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ddd915b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322582080:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695476:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582528 00000100:00100000:1.0:1761315753.695504:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315753.695527:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1548 00000100:00100000:1.0:1761315753.695532:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322582464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695541:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322582528:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315753.695567:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315753.695629:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322582528:12345-0@lo:400:kworker/u8:2.0 Request processed in 91us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315753.695634:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2337 00000100:00100000:1.0:1761315753.695647:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322582336:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695658:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1548 00000100:00100000:0.0:1761315753.695706:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582144 00000100:00100000:0.0:1761315753.695731:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582400 00000100:00100000:0.0:1761315753.695750:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582080 00010000:02020000:1.0:1761315753.695757:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315753.695762:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582464 00000100:00100000:0.0:1761315753.695780:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4749 00000100:00100000:0.0:1761315753.695785:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322582144:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315753.695845:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322582336:12345-0@lo:8:kworker/u8:2.0 Request processed in 196us (674us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315753.695856:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2337 00000100:00100000:0.0:1761315753.695866:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ca88bcf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322582144:12345-0@lo:400:kworker/u8:2.0 Request processed in 78us (814us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315753.695877:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4749 00000100:00100000:0.0:1761315753.695913:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4750 00000100:00100000:0.0:1761315753.695918:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e403e677 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322582400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.695953:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e403e677 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322582400:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315753.696552:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315753.696855:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4751 00000100:00100000:0.0:1761315753.696867:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4750 00000100:00100000:1.0:1761315753.696868:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc2cdad8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322582080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315753.696881:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4752 00000100:00100000:2.0:1761315753.696891:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322582464:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.696929:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322582144:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.697092:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000065048840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322582528:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315753.697206:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322582272:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315753.697300:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1761315753.697861:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315753.697866:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315753.697871:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2d157aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322582336:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315753.697913:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ddd915b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322582080:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.697923:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002db6a69f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322582464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.697931:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b0c2887 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322582400:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.698002:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582016 00000100:00100000:0.0:1761315753.698037:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1752 00000100:00100000:0.0:1761315753.698041:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322582016:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.700181:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322582016:12345-0@lo:400:kworker/u8:2.0 Request processed in 2132us (4908us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315753.700196:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1752 00000100:00100000:0.0:1761315753.700255:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582208 00000100:00100000:0.0:1761315753.700284:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2968 00000100:00100000:0.0:1761315753.700288:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322582208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315753.700336:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322582208:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (3786us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315753.700358:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2968 00000100:00100000:0.0:1761315753.700417:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008250083e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322582016:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315753.700534:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322582272:12345-0@lo:8:kworker/u8:2.0 Request processed in 3330us (5410us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315753.700543:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2336 00000100:00100000:3.0:1761315753.700587:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047e6259e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322582208:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315753.701241:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc2cdad8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322582080:12345-0@lo:400:kworker/u8:2.0 Request processed in 4374us (5736us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315753.701252:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4751 00000100:00100000:2.0:1761315753.701336:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a3d67ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322582464:12345-0@lo:400:kworker/u8:2.0 Request processed in 4446us (5769us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315753.701355:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4752 00000100:00080000:0.0:1761315753.702662:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315753.702670:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315753.702675:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322582272:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315754.715597:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315755.721854:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315755.721905:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315755.722394:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564544 00000100:00100000:0.0:1761315755.722446:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1753 00000100:00100000:0.0:1761315755.722453:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307564544:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315755.722521:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307564544:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 68us (676us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315755.722528:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1753 00000100:00100000:1.0:1761315755.723233:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564672 00000100:00100000:1.0:1761315755.723277:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4753 00000100:00100000:1.0:1761315755.723282:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307564672:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315755.723318:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a2767f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307564672:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 36us (1414us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315755.723325:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4753 00000100:00100000:2.0:1761315755.723857:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315755.723984:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315755.724036:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564608 00000100:00100000:1.0:1761315755.724072:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4754 00000100:00100000:1.0:1761315755.724077:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307564608:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315755.724111:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f210d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307564608:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315755.724116:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4754 00000100:00100000:3.0:1761315755.724681:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564736 00000100:00100000:3.0:1761315755.725460:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2338 00000100:00100000:3.0:1761315755.725465:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307564736:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315755.725558:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315755.725612:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307564736:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 146us (1628us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315755.725619:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2338 00010000:00010000:1.0:1761315755.740671:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315756.251887:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322582592:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315756.251951:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315756.252064:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582592 00000100:00100000:1.0:1761315756.252113:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2969 00000100:00100000:1.0:1761315756.252119:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322582592:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315756.252152:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315756.252200:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322582592:12345-0@lo:41:osp-pre-1-0.0 Request processed in 81us (250us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315756.252208:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2969 00000100:00100000:3.0:1761315756.252286:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322582592:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315756.764304:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315757.788683:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315758.811497:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed3a4f84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322582656:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315758.811562:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1761315758.811610:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315758.811870:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315758.811898:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582656 00000100:00100000:3.0:1761315758.812043:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322582720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315758.812441:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322582784:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315758.812480:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315758.812487:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761315758.812487:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315758.812492:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315758.812498:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1761315758.812501:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315758.812506:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315758.812513:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315758.812521:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322582848:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315758.812527:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060d3b1bb x1846869322583040/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315758.812543:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315758.812546:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315758.812549:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761315758.812550:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315758.812556:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4001 00000100:00080000:1.0:1761315758.812558:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315758.812562:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315758.812573:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000500f063b x1846869322583104/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315758.812597:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582784 00000100:00100000:2.0:1761315758.812633:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1754 00000100:00100000:1.0:1761315758.812635:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322583232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.812642:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322582784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315758.812667:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315758.812692:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315758.812697:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322582784:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315758.812699:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322583168:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.812708:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1754 00000100:00100000:1.0:1761315758.812743:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315758.812760:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322583040:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315758.812787:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582848 00000100:00100000:2.0:1761315758.812814:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315758.812816:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583232 00000100:00100000:2.0:1761315758.812833:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322583104:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315758.812852:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583168 00000100:00100000:2.0:1761315758.812863:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315758.812891:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4755 00000100:00100000:1.0:1761315758.812896:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322582848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.812900:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4756 00000100:00100000:2.0:1761315758.812907:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322583232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315758.812947:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2770fdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322582848:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (399us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315758.812953:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a254dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322583232:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315758.812956:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4755 00000100:00100000:2.0:1761315758.812965:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4756 00000100:00100000:1.0:1761315758.812977:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4757 00000100:00100000:1.0:1761315758.812982:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322583168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.812998:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583040 00000100:00100000:1.0:1761315758.813014:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a0c7468 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322583168:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315758.813018:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583104 00000100:00100000:1.0:1761315758.813023:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4757 00000100:00100000:2.0:1761315758.813035:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2339 00000100:00100000:2.0:1761315758.813040:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002027df4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322583040:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315758.813056:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2340 00000100:00100000:1.0:1761315758.813060:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322583104:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315758.813156:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315758.813187:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002027df4b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322583040:12345-0@lo:8:kworker/u8:0.0 Request processed in 146us (375us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315758.813193:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2339 00000100:00100000:3.0:1761315758.813237:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315758.813259:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322582912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813278:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315758.813289:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322583296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813307:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315758.813312:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315758.813316:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315758.813319:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322583040:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813320:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315758.813325:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322582976:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813359:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315758.813363:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583296 00000100:00100000:2.0:1761315758.813385:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1549 00000100:00100000:2.0:1761315758.813389:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322583296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.813413:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322583296:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (106us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315758.813418:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1549 00000100:00100000:2.0:1761315758.813449:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582912 00000100:00100000:2.0:1761315758.813473:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4758 00000100:00100000:2.0:1761315758.813477:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067d9ad19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322582912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315758.813502:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067d9ad19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322582912:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (224us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315758.813507:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4758 00000100:00100000:2.0:1761315758.813531:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322583168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315758.813534:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582976 00000100:00100000:0.0:1761315758.813557:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2970 00000100:00100000:0.0:1761315758.813561:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322582656:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315758.813583:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315758.813621:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322583360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813642:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315758.813659:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322582912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813667:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322583296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813691:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2971 00000100:00100000:3.0:1761315758.813695:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322582976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813723:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322582976:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (365us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315758.813729:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2971 00000100:00100000:3.0:1761315758.813747:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583360 00000100:00100000:3.0:1761315758.813778:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1550 00000100:00100000:3.0:1761315758.813781:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322583360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813808:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322583360:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (166us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315758.813813:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1550 00000100:00100000:3.0:1761315758.813834:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322582976:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813842:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322583360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813881:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322582848:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813890:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322582784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315758.813940:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322582720 00000100:00100000:3.0:1761315758.813961:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1027 00000100:00100000:3.0:1761315758.813965:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322582720:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315758.813983:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315758.814013:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322582720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (775us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315758.814018:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1027 00000100:00100000:3.0:1761315758.814047:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322582720:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315758.815242:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322583232:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315758.816580:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322582656:12345-0@lo:41:osp-pre-0-1.0 Request processed in 3013us (5014us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315758.816594:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2970 00000100:00100000:3.0:1761315758.816716:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed3a4f84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322582656:0@lo:41:osp-pre-0-1.0 00010000:02020000:1.0:1761315758.820075:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315758.820172:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322583104:12345-0@lo:8:kworker/u8:0.0 Request processed in 7109us (7308us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315758.820185:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2340 00000100:00080000:2.0:1761315758.820248:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315758.820252:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315758.820256:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322583104:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315759.835536:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315760.845933:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315760.845984:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315760.846034:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315760.846067:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315760.846157:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564864 00000100:00100000:0.0:1761315760.846205:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564928 00000100:00100000:0.0:1761315760.846227:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4759 00000100:00100000:0.0:1761315760.846235:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307564864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315760.846306:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003046dc2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307564864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 71us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315760.846314:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4759 00000100:00100000:0.0:1761315760.846328:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4760 00000100:00100000:0.0:1761315760.846331:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307564928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315760.846374:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a14c3afa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307564928:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (310us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315760.846380:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4760 00000100:00100000:0.0:1761315760.846395:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564800 00000100:00100000:0.0:1761315760.846414:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1755 00000100:00100000:0.0:1761315760.846417:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307564800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315760.846436:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307564800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 18us (404us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315760.846442:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1755 00000100:00100000:1.0:1761315760.846621:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307564992 00000100:00100000:1.0:1761315760.846659:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2341 00000100:00100000:1.0:1761315760.846664:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307564992:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315760.846765:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315760.848063:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307564992:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 1394us (2135us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315760.848075:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2341 00010000:00010000:0.0:1761315760.859881:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315761.373724:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315761.373757:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322583424:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315761.373829:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315761.374252:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583424 00000100:00100000:3.0:1761315761.374310:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2972 00000100:00100000:3.0:1761315761.374316:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322583424:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315761.374373:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315761.374424:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322583424:12345-0@lo:41:osp-pre-1-0.0 Request processed in 107us (596us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315761.374431:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2972 00000100:00100000:1.0:1761315761.374759:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322583424:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315761.883654:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315762.907630:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315763.931458:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315763.931755:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea8b91f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322583488:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315763.931825:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315763.932393:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315763.932398:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315763.932401:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315763.932405:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4006 00000100:00080000:0.0:1761315763.932408:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315763.932412:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315763.932427:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000496ea1dd x1846869322583808/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315763.932442:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315763.932444:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315763.932445:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315763.932447:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4006 00000100:00080000:0.0:1761315763.932448:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315763.932451:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315763.932457:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004e9b55f3 x1846869322583872/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315763.932501:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322583552:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.932534:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315763.932559:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322583936:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.932578:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315763.932596:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315763.932601:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed3a4f84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322583616:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.932623:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315763.932638:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322584000:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.932657:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315763.932690:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583552 00000100:00100000:0.0:1761315763.932745:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583936 00000100:00100000:3.0:1761315763.932790:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d8880f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322583680:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.932819:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315763.932835:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047e6259e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322584064:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315763.932903:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583488 00000100:00100000:2.0:1761315763.932935:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1756 00000100:00100000:2.0:1761315763.932940:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322583488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.933769:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315763.933795:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315763.933802:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322583744:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.933827:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315763.933842:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008250083e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322584128:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.933863:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315763.933927:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584000 00000100:00100000:3.0:1761315763.933952:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583680 00000100:00100000:3.0:1761315763.933964:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4761 00000100:00100000:3.0:1761315763.933969:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b0c2887 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322583552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934007:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b0c2887 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322583552:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1472us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934015:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4761 00000100:00100000:1.0:1761315763.934025:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4762 00000100:00100000:1.0:1761315763.934031:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322584000:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934037:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4763 00000100:00100000:3.0:1761315763.934045:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b6424f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322583680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315763.934084:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2d157aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322584000:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (1425us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315763.934094:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4762 00000100:00100000:3.0:1761315763.934094:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b6424f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322583680:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1271us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934103:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4763 00000100:00100000:3.0:1761315763.934139:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584064 00000100:00100000:1.0:1761315763.934145:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583616 00000100:00100000:3.0:1761315763.934167:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584128 00000100:00100000:1.0:1761315763.934178:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583744 00000100:00100000:3.0:1761315763.934194:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1551 00000100:00100000:3.0:1761315763.934199:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322584064:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315763.934206:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2973 00000100:00100000:1.0:1761315763.934212:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322583616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934230:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322584064:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (462us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934237:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1551 00000100:00100000:3.0:1761315763.934252:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1552 00000100:00100000:3.0:1761315763.934256:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322584128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315763.934263:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322583616:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (1638us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315763.934271:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2973 00000100:00100000:3.0:1761315763.934291:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322584128:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (425us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315763.934292:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2974 00000100:00100000:1.0:1761315763.934298:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322583744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934303:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1552 00000100:00100000:1.0:1761315763.934333:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322583744:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (505us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934337:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315763.934340:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2974 00000100:00100000:3.0:1761315763.934362:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322584192:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315763.934392:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315763.934420:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584192 00000100:00100000:3.0:1761315763.934466:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1028 00000100:00100000:1.0:1761315763.934470:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed3a4f84 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322583616:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934472:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322584192:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315763.934487:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322584000:0@lo:400:kworker/u8:0.0 00002000:00000024:3.0:1761315763.934496:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315763.934502:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322583552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934525:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322584192:12345-0@lo:13:osp-pre-1-1.0 Request processed in 53us (134us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934531:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1028 00000100:00100000:3.0:1761315763.934554:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004d9b4586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322584256:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315763.934581:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315763.934591:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4764 00000100:00100000:0.0:1761315763.934598:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322583936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934609:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322584192:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315763.934638:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584256 00000100:00100000:0.0:1761315763.934646:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002db6a69f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322583936:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (2067us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315763.934654:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4764 00000100:00100000:3.0:1761315763.934665:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2975 00000100:00100000:3.0:1761315763.934669:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322584256:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315763.934689:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315763.934693:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322583744:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.934705:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008250083e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322584128:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.934720:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d8880f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322583680:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934736:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322584256:12345-0@lo:41:osp-pre-0-1.0 Request processed in 55us (145us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315763.934744:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2975 00000100:00100000:0.0:1761315763.934745:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047e6259e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322584064:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.934781:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004d9b4586 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322584256:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315763.934794:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322583808:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315763.934823:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315763.934930:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322583488:12345-0@lo:400:kworker/u8:0.0 Request processed in 1988us (3104us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315763.934939:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1756 00000100:00100000:2.0:1761315763.935038:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea8b91f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322583488:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315763.935451:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322583872:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315763.935477:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315763.935582:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322583936:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315763.935957:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583808 00000100:00100000:3.0:1761315763.936001:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322583872 00000100:00100000:2.0:1761315763.936017:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2342 00000100:00100000:2.0:1761315763.936022:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322583808:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315763.936035:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2343 00000100:00100000:3.0:1761315763.936039:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322583872:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315763.936227:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315763.936267:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322583872:12345-0@lo:8:kworker/u8:0.0 Request processed in 226us (790us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315763.936274:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2343 00000100:00080000:0.0:1761315763.936458:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315763.936462:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315763.936467:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322583872:0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315763.936837:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315763.936890:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322583808:12345-0@lo:8:kworker/u8:0.0 Request processed in 866us (2067us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315763.936897:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2342 00000100:00080000:0.0:1761315763.936995:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315763.936999:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315763.937002:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322583808:0@lo:8:kworker/u8:0.0 00010000:00010000:1.0:1761315764.956209:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315765.962028:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315765.962070:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315765.962101:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315765.962149:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565056 00000100:00100000:2.0:1761315765.962167:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315765.962195:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565248 00000100:00100000:1.0:1761315765.962202:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1757 00000100:00100000:1.0:1761315765.962210:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307565056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315765.962222:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2344 00000100:00100000:2.0:1761315765.962227:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307565248:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761315765.962276:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307565056:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 66us (206us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315765.962285:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1757 00010000:02020000:2.0:1761315765.962319:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315765.962375:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307565248:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 146us (209us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315765.962383:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565120 00000100:00100000:2.0:1761315765.962383:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2344 00000100:00100000:0.0:1761315765.962430:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565184 00000100:00100000:0.0:1761315765.962460:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4765 00000100:00100000:0.0:1761315765.962465:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307565120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315765.962495:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4766 00000100:00100000:0.0:1761315765.962504:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b91fac8e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307565120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 38us (483us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315765.962510:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307565184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315765.962517:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4765 00000100:00100000:3.0:1761315765.962544:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046eace05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307565184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (442us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315765.962552:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4766 00010000:00010000:0.0:1761315765.979596:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315766.493384:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322584320:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315766.493445:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315766.493534:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584320 00000100:00100000:3.0:1761315766.493580:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2976 00000100:00100000:3.0:1761315766.493585:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322584320:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315766.493613:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315766.493654:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322584320:12345-0@lo:41:osp-pre-1-0.0 Request processed in 68us (209us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315766.493660:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2976 00000100:00100000:0.0:1761315766.493756:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322584320:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315767.005272:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315768.027523:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315769.051626:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322584384:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315769.051706:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.051811:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584384 00000100:00100000:3.0:1761315769.051863:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1029 00000100:00100000:3.0:1761315769.051869:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322584384:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315769.051898:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315769.051938:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322584384:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.051945:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1029 00000100:00100000:2.0:1761315769.052077:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322584384:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315769.052557:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315769.052660:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315769.052664:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315769.052667:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315769.052672:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4011 00000100:00080000:2.0:1761315769.052674:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315769.052678:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315769.052701:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060d3b1bb x1846869322584768/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315769.052712:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315769.052715:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315769.052716:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315769.052718:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4011 00000100:00080000:2.0:1761315769.052720:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315769.052722:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315769.052730:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000327d56cf x1846869322584832/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315769.052793:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322584640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.052826:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315769.052853:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585024:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.052878:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315769.052897:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315769.052902:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322584576:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.052922:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315769.052934:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322584960:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.052949:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315769.052971:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584640 00000100:00100000:2.0:1761315769.053000:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584960 00000100:00100000:2.0:1761315769.053016:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4767 00000100:00100000:2.0:1761315769.053020:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322584640:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.053059:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082ed9ce1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322584640:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (233us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315769.053066:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4767 00000100:00100000:2.0:1761315769.053078:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4768 00000100:00100000:2.0:1761315769.053081:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322584960:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.053103:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322584960:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (154us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315769.053108:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4768 00000100:00100000:2.0:1761315769.053123:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585024 00000100:00100000:2.0:1761315769.053138:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1553 00000100:00100000:2.0:1761315769.053141:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585024:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.053161:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585024:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315769.053166:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1553 00000100:00100000:2.0:1761315769.053190:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322584960:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.053198:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322584640:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315769.053204:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585024:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315769.053493:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584576 00000100:00100000:1.0:1761315769.053531:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2977 00000100:00100000:1.0:1761315769.053535:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322584576:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315769.053582:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322584576:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (659us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315769.053588:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2977 00000100:00100000:1.0:1761315769.053621:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322584576:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.053739:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322584512:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.053768:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.053793:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fed55ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322584896:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.053814:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.053831:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315769.053837:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322584448:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.053910:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.053927:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322584704:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.053949:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.053961:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322585088:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315769.053980:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315769.054005:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584448 00000100:00100000:3.0:1761315769.054029:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1758 00000100:00100000:3.0:1761315769.054033:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322584448:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054068:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322584448:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.054074:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1758 00000100:00100000:3.0:1761315769.054100:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584704 00000100:00100000:3.0:1761315769.054118:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585088 00000100:00100000:3.0:1761315769.054127:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2978 00000100:00100000:3.0:1761315769.054130:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322584704:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054155:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322584704:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.054160:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2978 00000100:00100000:3.0:1761315769.054169:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2979 00000100:00100000:3.0:1761315769.054172:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322585088:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315769.054193:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315769.054218:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322585088:12345-0@lo:41:osp-pre-0-1.0 Request processed in 45us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.054228:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2979 00000100:00100000:3.0:1761315769.054251:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584512 00000100:00100000:3.0:1761315769.054270:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584896 00000100:00100000:3.0:1761315769.054282:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4769 00000100:00100000:3.0:1761315769.054285:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322584512:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054308:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322584512:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (541us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.054313:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4769 00000100:00100000:3.0:1761315769.054321:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4770 00000100:00100000:3.0:1761315769.054324:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322584896:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054340:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322584896:12345-0@lo:400:kworker/u8:0.0 Request processed in 16us (526us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315769.054370:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4770 00000100:00100000:3.0:1761315769.054395:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322584448:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054402:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322584704:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054410:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322585088:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315769.054417:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322584512:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315769.054423:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fed55ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322584896:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315769.054499:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322584768:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315769.054526:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315769.054555:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322584832:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315769.054576:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315769.054611:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584768 00000100:00100000:0.0:1761315769.054627:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322584832 00000100:00100000:0.0:1761315769.054639:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2345 00000100:00100000:0.0:1761315769.054642:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322584768:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315769.054732:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315769.054761:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322584768:12345-0@lo:8:kworker/u8:0.0 Request processed in 118us (235us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315769.054766:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2345 00000100:00100000:0.0:1761315769.054776:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2346 00000100:00100000:0.0:1761315769.054778:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322584832:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315769.054851:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315769.054871:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322584832:12345-0@lo:8:kworker/u8:0.0 Request processed in 91us (295us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315769.054876:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2346 00000100:00080000:0.0:1761315769.054901:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315769.054905:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315769.054907:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322584768:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315769.054914:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315769.054916:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315769.054918:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322584832:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315770.075526:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315771.079657:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315771.079797:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565312 00000100:00100000:2.0:1761315771.079810:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315771.079863:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315771.079894:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1759 00000100:00100000:3.0:1761315771.079901:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307565312:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315771.079980:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565440 00000100:00100000:3.0:1761315771.079986:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307565312:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 84us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315771.079996:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1759 00000100:00100000:1.0:1761315771.080016:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565376 00000100:00100000:1.0:1761315771.080083:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4771 00000100:00100000:1.0:1761315771.080088:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307565440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315771.080124:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307565440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315771.080130:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4771 00000100:00100000:1.0:1761315771.080143:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4772 00000100:00100000:1.0:1761315771.080147:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307565376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315771.080166:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307565376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 19us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315771.080171:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4772 00000100:00100000:2.0:1761315771.080533:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315771.080611:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565504 00000100:00100000:0.0:1761315771.080690:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2347 00000100:00100000:0.0:1761315771.080695:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307565504:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315771.080780:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315771.080858:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307565504:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 161us (334us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315771.080865:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2347 00010000:00010000:1.0:1761315771.101456:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315771.611889:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dbb03c9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322585152:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315771.611959:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315771.612196:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585152 00000100:00100000:1.0:1761315771.612248:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2980 00000100:00100000:1.0:1761315771.612254:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322585152:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315771.612285:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315771.612362:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322585152:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (383us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315771.612370:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2980 00000100:00100000:0.0:1761315771.612945:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dbb03c9d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322585152:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315772.125323:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315773.147503:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315774.171569:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315774.171573:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585344:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315774.171576:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315774.171580:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315774.171587:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4016 00000100:00080000:2.0:1761315774.171589:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315774.171595:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315774.171616:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a5918643 x1846869322585536/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315774.171630:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315774.171633:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315774.171635:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315774.171637:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4016 00000100:00080000:2.0:1761315774.171639:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315774.171643:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315774.171644:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315774.171672:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000500f063b x1846869322585600/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315774.171710:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322585728:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315774.171740:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315774.171753:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585344 00000100:00100000:3.0:1761315774.171761:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315774.171767:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322585408:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315774.171792:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315774.171796:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2981 00000100:00100000:2.0:1761315774.171804:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322585344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315774.171808:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fed55ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585792:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315774.171834:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315774.171863:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322585344:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (221us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315774.171868:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585344:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315774.171873:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2981 00000100:00100000:1.0:1761315774.171967:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001989a362 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322585280:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.171992:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172016:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322585664:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172033:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172051:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e351e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322585920:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315774.172066:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172079:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:1.0:1761315774.172083:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed1079bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322585216:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172106:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172116:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585472:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172130:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172139:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585856:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172154:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.172172:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585920 00000100:00100000:1.0:1761315774.172190:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585472 00000100:00100000:1.0:1761315774.172199:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2982 00000100:00100000:1.0:1761315774.172203:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322585920:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315774.172225:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315774.172257:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322585920:12345-0@lo:41:osp-pre-0-1.0 Request processed in 53us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.172263:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2982 00000100:00100000:1.0:1761315774.172276:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2983 00000100:00100000:1.0:1761315774.172280:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172304:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585472:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.172309:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2983 00000100:00100000:1.0:1761315774.172326:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585728 00000100:00100000:1.0:1761315774.172377:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585408 00000100:00100000:1.0:1761315774.172393:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585280 00000100:00100000:1.0:1761315774.172403:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585664 00000100:00100000:1.0:1761315774.172412:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4773 00000100:00100000:1.0:1761315774.172415:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c833dcbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322585728:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172445:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c833dcbf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322585728:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (706us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.172451:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4773 00000100:00100000:1.0:1761315774.172461:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4774 00000100:00100000:1.0:1761315774.172465:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dd134a6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322585408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.172491:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dd134a6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322585408:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (699us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315774.172497:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4775 00000100:00100000:2.0:1761315774.172502:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003cf904e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322585280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315774.172530:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003cf904e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322585280:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (538us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.173162:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4774 00000100:00100000:2.0:1761315774.173170:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4775 00000100:00100000:2.0:1761315774.173191:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4776 00000100:00100000:2.0:1761315774.173197:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000bdd257c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322585664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.173210:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585216 00000100:00100000:2.0:1761315774.173232:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000bdd257c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322585664:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (1198us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315774.173240:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4776 00000100:00100000:1.0:1761315774.173244:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1760 00000100:00100000:1.0:1761315774.173250:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322585216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315774.173288:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322585216:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (1180us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315774.173288:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585984:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315774.173300:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1760 00000100:00100000:0.0:1761315774.173321:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322585536:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315774.173329:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315774.173334:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585792 00000100:00100000:1.0:1761315774.173377:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585856 00000100:00100000:1.0:1761315774.173394:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1554 00000100:00100000:1.0:1761315774.173411:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7ad9d26 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322585792:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315774.173411:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585984 00000100:00100000:3.0:1761315774.173593:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1030 00000100:00100000:3.0:1761315774.173597:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322585984:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315774.173615:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315774.174539:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315774.174574:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322585600:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315774.174593:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7ad9d26 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322585792:12345-0@lo:400:kworker/u8:2.0 Request processed in 1178us (2756us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.174604:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1554 00000100:00100000:0.0:1761315774.174608:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315774.174677:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322585408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315774.174692:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fed55ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585792:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315774.174703:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322585728:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.174762:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585536 00000100:00100000:1.0:1761315774.174768:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322585600 00000100:00100000:0.0:1761315774.174790:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1555 00000100:00100000:0.0:1761315774.174795:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585856:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.174838:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322585856:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (2683us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315774.174872:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2348 00000100:00100000:1.0:1761315774.174878:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322585536:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315774.174968:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315774.175005:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322585536:12345-0@lo:8:kworker/u8:2.0 Request processed in 127us (469us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315774.175011:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2348 00000100:00100000:2.0:1761315774.175085:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2349 00000100:00100000:0.0:1761315774.175093:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1555 00000100:00100000:2.0:1761315774.175093:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095078c5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322585600:12345-0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315774.175134:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315774.175139:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315774.175142:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322585536:0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315774.175175:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315774.175215:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095078c5a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322585600:12345-0@lo:8:kworker/u8:2.0 Request processed in 123us (608us total) trans 0 rc -19/-19 00010000:00010000:0.0:1761315774.175216:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315774.175223:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2349 00000100:00080000:0.0:1761315774.175230:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315774.175234:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315774.175239:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322585600:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175276:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed1079bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322585216:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175289:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585472:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175299:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322585856:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175308:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001989a362 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322585280:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175316:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322585664:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315774.175329:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e351e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322585920:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315774.175545:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322585984:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1946us (2217us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315774.175554:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1030 00000100:00100000:2.0:1761315774.175582:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322585984:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315775.200473:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315776.201510:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315776.201560:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315776.201594:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315776.201623:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565568 00000100:00100000:1.0:1761315776.201749:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565632 00000100:00100000:3.0:1761315776.201761:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565696 00000100:00100000:1.0:1761315776.201785:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4777 00000100:00100000:1.0:1761315776.201791:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed3a4f84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307565632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315776.201800:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4778 00000100:00100000:3.0:1761315776.201806:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea8b91f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307565696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315776.201863:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea8b91f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307565696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 55us (268us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315776.201870:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4778 00000100:00100000:1.0:1761315776.201875:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed3a4f84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307565632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 82us (370us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315776.201883:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4777 00000100:00100000:0.0:1761315776.202702:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1761 00000100:00100000:0.0:1761315776.202709:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307565568:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315776.202742:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307565568:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (1183us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315776.202748:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1761 00000100:00100000:0.0:1761315776.205048:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315776.205767:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565760 00000100:00100000:1.0:1761315776.205818:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2350 00000100:00100000:1.0:1761315776.205824:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307565760:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315776.205914:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315776.206270:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307565760:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 444us (1228us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315776.206279:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2350 00010000:00010000:0.0:1761315776.219499:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315776.732450:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315776.732471:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322586048:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315776.732538:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315776.732664:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586048 00000100:00100000:1.0:1761315776.732719:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2984 00000100:00100000:1.0:1761315776.732725:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322586048:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315776.732758:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315776.732817:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322586048:12345-0@lo:41:osp-pre-1-0.0 Request processed in 91us (279us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315776.732824:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2984 00000100:00100000:3.0:1761315776.732936:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322586048:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315777.243526:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315778.267508:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315779.291544:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315779.291808:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322586112:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315779.291874:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315779.291974:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586112 00000100:00100000:1.0:1761315779.292029:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2985 00000100:00100000:1.0:1761315779.292035:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322586112:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315779.292071:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315779.292115:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322586112:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (243us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315779.292122:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2985 00000100:00100000:0.0:1761315779.292208:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322586112:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315779.292647:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315779.292652:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315779.292655:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315779.292661:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4021 00000100:00080000:3.0:1761315779.292663:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315779.292667:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315779.292685:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bb399f09 x1846869322586432/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315779.292697:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315779.292700:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315779.292701:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315779.292704:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4021 00000100:00080000:3.0:1761315779.292706:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315779.292709:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315779.292719:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000604db0f2 x1846869322586496/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315779.292875:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322586240:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.292911:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.292927:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cdd78470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322586624:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.292949:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.292966:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315779.292971:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000de1f52d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322586304:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.292990:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293004:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093e23d65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322586688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293026:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293053:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586240 00000100:00100000:3.0:1761315779.293079:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586624 00000100:00100000:3.0:1761315779.293095:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586304 00000100:00100000:3.0:1761315779.293112:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4779 00000100:00100000:3.0:1761315779.293116:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322586240:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293150:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfe7af88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322586240:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (239us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293156:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4779 00000100:00100000:3.0:1761315779.293169:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4780 00000100:00100000:3.0:1761315779.293172:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322586624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293197:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ffb7ac63 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322586624:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293202:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4780 00000100:00100000:3.0:1761315779.293211:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4781 00000100:00100000:3.0:1761315779.293214:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322586304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293235:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef192eeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322586304:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (245us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293240:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4781 00000100:00100000:3.0:1761315779.293261:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586688 00000100:00100000:3.0:1761315779.293281:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1556 00000100:00100000:3.0:1761315779.293284:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbb03c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322586688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293310:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbb03c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322586688:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293315:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1556 00000100:00100000:3.0:1761315779.293374:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000de1f52d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322586304:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293388:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093e23d65 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322586688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293394:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322586240:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293401:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cdd78470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322586624:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293427:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322586176:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293454:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293469:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322586560:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293485:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293499:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315779.293503:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322586368:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293525:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293539:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322586752:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293555:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315779.293576:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586176 00000100:00100000:3.0:1761315779.293597:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1762 00000100:00100000:3.0:1761315779.293601:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322586176:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293645:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322586176:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (191us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293654:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1762 00000100:00100000:3.0:1761315779.293677:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586560 00000100:00100000:3.0:1761315779.293696:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4782 00000100:00100000:3.0:1761315779.293699:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cdd78470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322586560:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293724:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cdd78470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322586560:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293729:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4782 00000100:00100000:3.0:1761315779.293746:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586752 00000100:00100000:3.0:1761315779.293763:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1557 00000100:00100000:3.0:1761315779.293766:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322586752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293789:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322586752:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315779.293794:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1557 00000100:00100000:3.0:1761315779.293816:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322586752:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293823:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322586176:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315779.293832:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322586560:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315779.294457:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586368 00000100:00100000:0.0:1761315779.294502:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2986 00000100:00100000:0.0:1761315779.294507:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322586368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315779.294557:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322586368:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (1032us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315779.294563:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2986 00000100:00100000:0.0:1761315779.294631:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322586432:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315779.294658:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315779.294673:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322586496:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315779.294692:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315779.294717:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586432 00000100:00100000:0.0:1761315779.294735:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586496 00000100:00100000:0.0:1761315779.294749:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2351 00000100:00100000:0.0:1761315779.294753:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322586432:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315779.294839:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315779.294869:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322586432:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (211us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315779.294875:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2351 00000100:00100000:0.0:1761315779.294886:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2352 00000100:00100000:0.0:1761315779.294889:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322586496:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315779.294967:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315779.294988:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322586496:12345-0@lo:8:kworker/u8:2.0 Request processed in 98us (296us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315779.294994:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2352 00000100:00080000:0.0:1761315779.295025:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315779.295029:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315779.295032:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322586432:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315779.295043:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315779.295046:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315779.295048:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322586496:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315779.295067:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315779.295073:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322586816:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315779.295099:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315779.295126:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586816 00000100:00100000:0.0:1761315779.295147:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1031 00000100:00100000:0.0:1761315779.295151:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322586816:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315779.295171:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315779.295202:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322586816:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (103us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315779.295209:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1031 00000100:00100000:0.0:1761315779.295233:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322586816:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315779.299741:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322586368:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315780.316204:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315781.327711:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315781.327771:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315781.327806:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315781.327889:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315781.327930:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566016 00000100:00100000:0.0:1761315781.327981:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2353 00000100:00100000:0.0:1761315781.327988:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f13d776a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307566016:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315781.328079:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315781.328148:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f13d776a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307566016:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 160us (259us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315781.328156:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2353 00000100:00100000:2.0:1761315781.328374:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565888 00000100:00100000:2.0:1761315781.328419:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565952 00000100:00100000:2.0:1761315781.328439:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4783 00000100:00100000:2.0:1761315781.328443:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c8626f82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307565888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315781.328489:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c8626f82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307565888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 45us (785us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315781.328496:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4783 00000100:00100000:2.0:1761315781.328508:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4784 00000100:00100000:2.0:1761315781.328512:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5d2f229 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307565952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315781.328533:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5d2f229 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307565952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (727us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315781.328539:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4784 00000100:00100000:1.0:1761315781.328663:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307565824 00000100:00100000:1.0:1761315781.328700:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1763 00000100:00100000:1.0:1761315781.328705:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307565824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315781.328739:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307565824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (967us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315781.328745:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1763 00010000:00010000:0.0:1761315781.339517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315781.853116:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315781.853141:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322586880:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315781.853209:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315781.853481:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586880 00000100:00100000:2.0:1761315781.853537:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2987 00000100:00100000:2.0:1761315781.853543:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322586880:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315781.853577:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315781.853640:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322586880:12345-0@lo:41:osp-pre-1-0.0 Request processed in 96us (431us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315781.853648:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2987 00000100:00100000:1.0:1761315781.853769:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322586880:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315782.366492:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315783.387645:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315784.411723:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315784.411731:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315784.411735:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315784.411749:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4026 00000100:00080000:2.0:1761315784.411753:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315784.411760:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315784.411783:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060c89ebd x1846869322587264/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315784.411793:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322586944:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315784.411803:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315784.411808:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315784.411810:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315784.411815:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4026 00000100:00080000:2.0:1761315784.411818:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315784.411821:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315784.411836:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bd36f033 x1846869322587456/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315784.411841:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060c89ebd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322587264:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315784.411874:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315784.411890:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315784.411906:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322587200:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315784.411912:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322586944 00000100:00100000:0.0:1761315784.411919:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd36f033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322587456:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315784.411938:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315784.411951:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315784.411960:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1764 00000100:00100000:2.0:1761315784.411967:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322586944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.411969:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322587584:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315784.411989:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587264 00000100:00100000:1.0:1761315784.412006:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315784.412015:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587456 00000100:00100000:2.0:1761315784.412024:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322586944:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (151us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315784.412035:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1764 00000100:00100000:0.0:1761315784.412040:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2354 00000100:00100000:1.0:1761315784.412043:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322586944:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315784.412047:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322587264:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315784.412065:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315784.412071:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322587136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.412094:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315784.412100:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2355 00000100:00100000:2.0:1761315784.412106:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049a2c7d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322587456:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315784.412112:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322587520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.412149:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:0.0:1761315784.412177:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315784.412190:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587584 00000100:00100000:1.0:1761315784.412219:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587136 00000100:00100000:0.0:1761315784.412226:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322587264:12345-0@lo:8:kworker/u8:2.0 Request processed in 179us (336us total) trans 0 rc -19/-19 00010000:02020000:2.0:1761315784.412231:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315784.412236:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2354 00000100:00100000:1.0:1761315784.412237:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587520 00000100:00100000:1.0:1761315784.412261:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4785 00000100:00100000:1.0:1761315784.412266:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322587584:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315784.412272:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315784.412272:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049a2c7d7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322587456:12345-0@lo:8:kworker/u8:2.0 Request processed in 164us (321us total) trans 0 rc -19/-19 00000100:00080000:0.0:1761315784.412278:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315784.412283:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2355 00000100:00100000:0.0:1761315784.412285:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd36f033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322587456:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315784.412301:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315784.412303:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315784.412308:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060c89ebd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322587264:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315784.412313:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587200 00000100:00100000:1.0:1761315784.412318:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001ce50f66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322587584:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (312us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315784.412326:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4785 00000100:00100000:2.0:1761315784.412386:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2988 00010000:00010000:0.0:1761315784.412390:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315784.412391:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322587200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315784.412420:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001d4f7a52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322587584:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315784.412436:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322587200:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (498us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315784.412440:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322587200:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.412444:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587072:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315784.412447:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2988 00000100:00100000:3.0:1761315784.412484:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412505:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587392:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315784.412525:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412539:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322587712:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.412562:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412591:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:3.0:1761315784.412598:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322587008:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.412620:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412629:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315784.412650:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412667:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587648:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.412688:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315784.412714:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587328 00000100:00100000:3.0:1761315784.412741:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1032 00000100:00100000:3.0:1761315784.412745:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322587328:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315784.412768:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315784.412801:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322587328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.412807:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1032 00000100:00100000:3.0:1761315784.412839:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587072 00000100:00100000:3.0:1761315784.412869:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587392 00000100:00100000:3.0:1761315784.412880:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2989 00000100:00100000:3.0:1761315784.412884:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322587072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.412909:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322587072:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (427us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.412915:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2989 00000100:00100000:3.0:1761315784.412925:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2990 00000100:00100000:3.0:1761315784.412928:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013ebcdeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322587392:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315784.412957:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315784.412983:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013ebcdeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322587392:12345-0@lo:41:osp-pre-0-1.0 Request processed in 54us (458us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.412988:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2990 00000100:00100000:3.0:1761315784.413007:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587712 00000100:00100000:3.0:1761315784.413026:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587648 00000100:00100000:3.0:1761315784.413043:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1558 00000100:00100000:3.0:1761315784.413046:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322587712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.413072:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322587712:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (510us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.413077:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1558 00000100:00100000:3.0:1761315784.413086:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1559 00000100:00100000:3.0:1761315784.413090:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322587648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.413124:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322587648:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (436us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.413129:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1559 00000100:00100000:3.0:1761315784.413160:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315784.413169:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587648:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.413179:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4786 00000100:00100000:3.0:1761315784.413180:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587072:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.413189:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322587136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315784.413194:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587392:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315784.413198:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4787 00000100:00100000:3.0:1761315784.413209:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0fc10e3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322587712:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315784.413238:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f86d06ac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322587136:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (1143us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315784.413242:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587008 00000100:00100000:1.0:1761315784.413246:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4786 00000100:00100000:0.0:1761315784.413268:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4788 00000100:00100000:2.0:1761315784.413291:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a10d07e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322587520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315784.413337:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a10d07e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322587520:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (1187us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315784.413371:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322587136:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315784.413377:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4787 00000100:00100000:3.0:1761315784.413389:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000444a05f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322587520:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315784.413449:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000de1f52d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322587008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315784.413499:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000de1f52d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322587008:12345-0@lo:400:kworker/u8:2.0 Request processed in 224us (877us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315784.413505:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4788 00000100:00100000:3.0:1761315784.413592:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322587008:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315785.435904:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315786.444997:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315786.445283:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315786.445383:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566080 00000100:00100000:0.0:1761315786.446308:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315786.446437:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1765 00000100:00100000:1.0:1761315786.446445:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307566080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315786.446512:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307566080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (1522us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315786.446519:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1765 00000100:00100000:0.0:1761315786.447700:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315786.447815:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566272 00000100:00100000:2.0:1761315786.447888:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2356 00000100:00100000:2.0:1761315786.447894:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092fa280b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307566272:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315786.447996:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315786.448036:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092fa280b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307566272:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 142us (347us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315786.448043:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2356 00000100:00100000:3.0:1761315786.448493:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566208 00000100:00100000:3.0:1761315786.448529:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566144 00000100:00100000:3.0:1761315786.448549:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4789 00000100:00100000:3.0:1761315786.448553:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005266f782 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307566208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315786.448604:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005266f782 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307566208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 50us (3322us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315786.448611:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4789 00000100:00100000:3.0:1761315786.448623:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4790 00000100:00100000:3.0:1761315786.448626:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000154bc3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307566144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315786.448647:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000154bc3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307566144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 20us (2343us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315786.448653:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4790 00010000:00010000:0.0:1761315786.459694:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315786.971636:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322587776:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315786.971710:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315786.971835:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587776 00000100:00100000:3.0:1761315786.972778:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2991 00000100:00100000:3.0:1761315786.972787:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b38f16ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322587776:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315786.972822:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315786.972886:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b38f16ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322587776:12345-0@lo:41:osp-pre-1-0.0 Request processed in 100us (1177us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315786.972893:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2991 00000100:00100000:0.0:1761315786.972989:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322587776:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315787.484084:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315788.507493:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315789.531492:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315789.531663:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322587840:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315789.531794:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.531900:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315789.531912:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000196be7fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587904:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315789.531942:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.531989:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587904 00000100:00100000:0.0:1761315789.532029:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2992 00000100:00100000:0.0:1761315789.532034:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009084fbbe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322587904:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315789.532062:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315789.532144:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315789.532163:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322587968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315789.532729:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322588032:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315789.532744:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315789.532749:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315789.532752:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315789.532757:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4031 00000100:00080000:3.0:1761315789.532759:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315789.532764:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315789.532782:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bca5f7bb x1846869322588224/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315789.532797:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315789.532799:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315789.532801:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315789.532803:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4031 00000100:00080000:3.0:1761315789.532806:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315789.532808:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315789.532821:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f9a1340f x1846869322588288/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315789.532869:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322588224:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315789.534338:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009084fbbe pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322587904:12345-0@lo:41:osp-pre-0-1.0 Request processed in 2298us (2391us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315789.534364:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2992 00000100:00100000:2.0:1761315789.534400:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.534408:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587840 00000100:00100000:0.0:1761315789.534445:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1766 00000100:00100000:0.0:1761315789.534452:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322587840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315789.534457:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322588416:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.534504:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322587840:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (2770us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315789.534505:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.534513:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1766 00000100:00100000:2.0:1761315789.534742:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315789.534749:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322588096:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.534834:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588032 00000100:00100000:3.0:1761315789.534851:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.534868:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4791 00000100:00100000:2.0:1761315789.534872:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.534876:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000daf1a052 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322588032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315789.534884:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322588288:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315789.534904:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322588480:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.534930:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000daf1a052 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322588032:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (530us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315789.534930:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.534938:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4791 00000100:00100000:0.0:1761315789.534953:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588416 00000100:00100000:0.0:1761315789.534976:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588096 00000100:00100000:3.0:1761315789.534978:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588224 00000100:00100000:3.0:1761315789.535004:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588288 00000100:00100000:3.0:1761315789.535019:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2357 00000100:00100000:3.0:1761315789.535023:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322588224:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315789.535036:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4792 00000100:00100000:0.0:1761315789.535041:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322588416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.535074:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000444a05f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322588416:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (572us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315789.535080:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4792 00000100:00100000:0.0:1761315789.535092:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4793 00000100:00100000:0.0:1761315789.535095:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322588096:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761315789.535110:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315789.535132:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0fc10e3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322588096:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (266us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315789.535141:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4793 00000100:00100000:3.0:1761315789.535156:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322588224:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (305us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315789.535163:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2357 00000100:00100000:0.0:1761315789.535173:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2358 00000100:00100000:0.0:1761315789.535182:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322588288:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315789.535260:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315789.535286:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322588288:12345-0@lo:8:kworker/u8:2.0 Request processed in 103us (363us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315789.535292:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2358 00000100:00100000:0.0:1761315789.535335:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322588352:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315789.535366:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.535574:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315789.535585:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322588160:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.535611:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000196be7fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322587904:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315789.535619:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315789.535642:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322588544:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315789.535645:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588352 00000100:00100000:1.0:1761315789.535671:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315789.535675:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322587968 00000100:00100000:1.0:1761315789.535704:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588160 00000100:00100000:3.0:1761315789.535733:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4794 00000100:00100000:1.0:1761315789.535737:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2993 00000100:00100000:3.0:1761315789.535740:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157d91fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322588352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315789.535745:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322588160:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315789.535797:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157d91fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322588352:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315789.535804:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4794 00000100:00100000:1.0:1761315789.535809:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322588160:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315789.535817:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2993 00000100:00100000:3.0:1761315789.535829:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588544 00000100:00100000:0.0:1761315789.535837:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1033 00000100:00100000:0.0:1761315789.535844:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322587968:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315789.535860:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1560 00000100:00100000:3.0:1761315789.535867:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322588544:12345-0@lo:400:kworker/u8:2.0 00002000:00000024:0.0:1761315789.535870:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315789.535906:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322588544:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (237us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315789.535912:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1560 00000100:00100000:0.0:1761315789.535916:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322587968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (573us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315789.535923:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1033 00000100:00100000:0.0:1761315789.535955:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322588352:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.535983:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322588160:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.535993:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322588544:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315789.536008:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322587968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315789.536020:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315789.536409:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322588416:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315789.536427:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315789.536428:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322588032:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315789.536434:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315789.536440:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322588224:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315789.536455:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315789.536458:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315789.536463:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322588288:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315789.536463:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322588096:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315789.536475:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588480 00000100:00100000:1.0:1761315789.536505:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1561 00000100:00100000:2.0:1761315789.536508:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322587840:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315789.536511:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322588480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315789.536796:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322588480:12345-0@lo:400:kworker/u8:2.0 Request processed in 285us (779us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315789.536804:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1561 00000100:00100000:2.0:1761315789.536856:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322588480:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315790.555677:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315791.563240:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315791.565851:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315791.566135:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566400 00000100:00100000:2.0:1761315791.566207:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4795 00000100:00100000:2.0:1761315791.566214:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000196be7fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307566400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315791.566294:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000196be7fc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307566400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 79us (3061us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315791.566301:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4795 00000100:00100000:0.0:1761315791.566548:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315791.567094:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566336 00000100:00100000:1.0:1761315791.567140:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1767 00000100:00100000:1.0:1761315791.567145:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307566336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315791.567185:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307566336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 39us (1342us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315791.567192:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1767 00000100:00100000:0.0:1761315791.567668:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315791.568326:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566528 00000100:00100000:1.0:1761315791.568392:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2359 00000100:00100000:1.0:1761315791.568396:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307566528:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315791.568486:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315791.568554:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307566528:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 156us (888us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315791.568561:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2359 00000100:00100000:2.0:1761315791.569025:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566464 00000100:00100000:2.0:1761315791.570860:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4796 00000100:00100000:2.0:1761315791.570872:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd31c3ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307566464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315791.570973:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd31c3ef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307566464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 101us (4425us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315791.570980:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4796 00010000:00010000:0.0:1761315791.580267:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315792.091744:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a16ccef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322588608:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315792.091808:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315792.092032:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588608 00000100:00100000:1.0:1761315792.092091:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2994 00000100:00100000:1.0:1761315792.092097:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322588608:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315792.092130:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315792.092372:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322588608:12345-0@lo:41:osp-pre-1-0.0 Request processed in 274us (564us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315792.092380:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2994 00000100:00100000:0.0:1761315792.092458:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a16ccef pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322588608:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315792.603602:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315793.627755:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315794.655969:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315794.655976:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315794.655979:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315794.655986:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4037 00000100:00080000:0.0:1761315794.655989:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315794.655995:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315794.656014:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000061659632 x1846869322589056/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315794.656041:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315794.656046:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315794.656047:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315794.656050:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4037 00000100:00080000:0.0:1761315794.656052:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315794.656056:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315794.656069:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f9a1340f x1846869322589120/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315794.656075:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322589056:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656145:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315794.656159:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322588864:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656184:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322589120:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656200:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315794.656215:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315794.656222:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322589248:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656253:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589056 00000100:00100000:0.0:1761315794.656257:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315794.656283:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315794.656286:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589120 00000100:00100000:0.0:1761315794.656291:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322588800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656306:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2360 00000100:00100000:1.0:1761315794.656312:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322589056:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656321:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315794.656338:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322589184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656380:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315794.656400:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315794.656411:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2361 00000100:00100000:0.0:1761315794.656415:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322589120:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315794.656447:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315794.656484:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322589056:12345-0@lo:8:kworker/u8:0.0 Request processed in 171us (340us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315794.656491:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2360 00010000:02020000:0.0:1761315794.656543:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:1.0:1761315794.656546:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315794.656550:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315794.656555:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000061659632 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322589056:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656573:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322589120:12345-0@lo:8:kworker/u8:0.0 Request processed in 156us (359us total) trans 0 rc -19/-19 00000100:00080000:1.0:1761315794.656580:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761315794.656581:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2361 00000100:00080000:1.0:1761315794.656586:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315794.656591:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9a1340f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322589120:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656611:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588864 00000100:00100000:1.0:1761315794.656638:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589248 00000100:00100000:0.0:1761315794.656641:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2995 00000100:00100000:0.0:1761315794.656646:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322588864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656671:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588800 00000100:00100000:0.0:1761315794.656690:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322588864:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (491us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315794.656693:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589184 00000100:00100000:0.0:1761315794.656699:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2995 00000100:00100000:1.0:1761315794.656718:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4797 00000100:00100000:1.0:1761315794.656723:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322589248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315794.656726:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4798 00000100:00100000:0.0:1761315794.656735:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a360afc6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322588800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656765:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002c906f54 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322589248:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (508us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315794.656771:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4797 00000100:00100000:0.0:1761315794.656781:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a360afc6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322588800:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (459us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315794.656788:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4799 00000100:00100000:1.0:1761315794.656792:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009bbe12bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322589184:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315794.656819:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009bbe12bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322589184:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (439us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315794.656948:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322588736:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315794.656981:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.656999:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322588992:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657021:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657033:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322589376:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657053:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657070:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1761315794.657076:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322588672:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657110:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657123:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322588928:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657143:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657152:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322589312:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657172:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657195:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322589440:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315794.657222:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315794.657250:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588992 00000100:00100000:2.0:1761315794.657270:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589440 00000100:00100000:2.0:1761315794.657279:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2996 00000100:00100000:2.0:1761315794.657282:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322588992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.657312:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322588992:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315794.657317:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2996 00000100:00100000:2.0:1761315794.657328:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2997 00000100:00100000:2.0:1761315794.657331:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322589440:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315794.657383:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315794.657429:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322589440:12345-0@lo:41:osp-pre-0-1.0 Request processed in 97us (207us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315794.657435:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2997 00000100:00100000:2.0:1761315794.657454:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588736 00000100:00100000:2.0:1761315794.657478:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1034 00000100:00100000:2.0:1761315794.657481:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000010dea4cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322588736:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315794.657499:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315794.657539:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000010dea4cd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322588736:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (558us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315794.657547:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1034 00000100:00100000:2.0:1761315794.657767:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588928 00000100:00100000:3.0:1761315794.659184:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322588672 00000100:00100000:3.0:1761315794.659231:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1768 00000100:00100000:3.0:1761315794.659236:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322588672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659298:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322588672:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (2188us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315794.659304:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1768 00000100:00100000:3.0:1761315794.659403:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589376 00000100:00100000:3.0:1761315794.659427:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589312 00000100:00100000:3.0:1761315794.659444:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1562 00000100:00100000:3.0:1761315794.659447:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322589376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659479:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322589376:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (2426us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315794.659485:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1562 00000100:00100000:3.0:1761315794.659494:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1563 00000100:00100000:3.0:1761315794.659497:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322589312:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659531:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322589312:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (2359us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315794.659536:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1563 00000100:00100000:3.0:1761315794.659571:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322588672:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659584:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322589312:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659598:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322588736:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315794.659607:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322588992:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659616:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322589376:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659668:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054511f4f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322589440:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315794.659739:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322588800:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659749:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322589184:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659757:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322588864:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315794.659766:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322589248:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315794.660964:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4798 00000100:00100000:1.0:1761315794.663887:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4799 00000100:00100000:2.0:1761315794.664774:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4800 00000100:00100000:2.0:1761315794.664785:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd36f033 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322588928:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315794.664892:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd36f033 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322588928:12345-0@lo:400:kworker/u8:0.0 Request processed in 110us (7748us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315794.664903:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4800 00000100:00100000:3.0:1761315794.665682:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322588928:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315795.675535:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315796.680606:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315796.680675:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566784 00000100:00100000:3.0:1761315796.680821:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2362 00000100:00100000:3.0:1761315796.680829:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307566784:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315796.680969:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315796.681036:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307566784:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 207us (440us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315796.681045:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2362 00000100:00100000:0.0:1761315796.681124:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315796.681736:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315796.682827:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566592 00000100:00100000:3.0:1761315796.682892:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1769 00000100:00100000:3.0:1761315796.682898:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307566592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315796.683189:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566720 00000100:00100000:2.0:1761315796.683217:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4801 00000100:00100000:2.0:1761315796.683222:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000012e48a9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307566720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315796.684757:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307566592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 1853us (3629us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315796.684770:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1769 00000100:00100000:0.0:1761315796.686258:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315796.686306:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566656 00000100:00100000:0.0:1761315796.686369:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4802 00000100:00100000:0.0:1761315796.686376:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c23e4cef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307566656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315796.686436:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c23e4cef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307566656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 60us (183us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315796.686444:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4802 00000100:00100000:2.0:1761315796.686820:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000012e48a9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307566720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 3596us (5084us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315796.686829:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4801 00010000:00010000:0.0:1761315796.699831:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315797.212163:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315797.212186:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322589504:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315797.212256:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315797.212932:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589504 00000100:00100000:3.0:1761315797.214021:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2998 00000100:00100000:3.0:1761315797.214031:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322589504:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315797.214069:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315797.214142:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322589504:12345-0@lo:41:osp-pre-1-0.0 Request processed in 111us (1886us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315797.214149:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2998 00000100:00100000:2.0:1761315797.214265:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322589504:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315797.724206:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315798.747670:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315799.771522:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322589696:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315799.771527:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315799.771532:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315799.771536:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315799.771541:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4042 00000100:00080000:0.0:1761315799.771544:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315799.771550:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315799.771572:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000765da965 x1846869322590016/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315799.771587:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b3a0ae9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322589760:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315799.771589:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315799.771595:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1761315799.771595:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315799.771597:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315799.771600:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4042 00000100:00080000:0.0:1761315799.771601:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315799.771604:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315799.771614:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1f55899 x1846869322590080/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315799.771626:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322589952:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315799.771639:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315799.771669:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315799.771679:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322590144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315799.771692:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322590336:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315799.771701:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322589824:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.771710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315799.771727:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315799.771813:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322589568:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315799.772600:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322590016:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315799.772678:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589696 00000100:00100000:3.0:1761315799.772785:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589760 00000100:00100000:3.0:1761315799.772813:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589952 00000100:00100000:3.0:1761315799.772831:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2999 00000100:00100000:3.0:1761315799.772836:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d479a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322589760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315799.772887:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d479a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322589760:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (1252us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315799.772894:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2999 00000100:00100000:3.0:1761315799.772911:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3000 00000100:00100000:3.0:1761315799.772915:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322589952:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315799.772979:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315799.773052:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322589952:12345-0@lo:41:osp-pre-0-1.0 Request processed in 135us (1384us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315799.773059:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3000 00000100:00100000:0.0:1761315799.773081:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315799.773109:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322590080:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315799.773118:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590336 00000100:00100000:0.0:1761315799.773134:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315799.773161:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315799.773170:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590016 00000100:00100000:0.0:1761315799.773177:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590080 00000100:00100000:0.0:1761315799.773208:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1564 00000100:00100000:2.0:1761315799.773209:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590144 00000100:00100000:0.0:1761315799.773217:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7f32d52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322590336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315799.773217:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2363 00000100:00100000:3.0:1761315799.773225:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322590016:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773233:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4803 00000100:00100000:2.0:1761315799.773238:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322589696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315799.773250:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7f32d52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322590336:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1524us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315799.773256:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1564 00000100:00100000:2.0:1761315799.773279:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a96fbe84 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322589696:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (1688us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315799.773284:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4803 00000100:00100000:2.0:1761315799.773294:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4804 00000100:00100000:2.0:1761315799.773298:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7952676 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322590144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773319:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7952676 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322590144:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (1611us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315799.773323:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4804 00000100:00100000:2.0:1761315799.773383:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322590144:0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761315799.773387:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315799.773396:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b3a0ae9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322589760:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315799.773403:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2364 00000100:00100000:0.0:1761315799.773407:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a07dadbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590080:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773449:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322590336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315799.773481:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322590016:12345-0@lo:8:kworker/u8:0.0 Request processed in 236us (382us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315799.773488:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2363 00010000:02020000:0.0:1761315799.773498:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315799.773533:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc64cfd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322589952:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315799.773558:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322589696:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315799.773562:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315799.773567:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315799.773573:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322590016:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773605:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315799.773632:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589824 00000100:00100000:2.0:1761315799.773640:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322590208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773669:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315799.773692:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590208 00000100:00100000:2.0:1761315799.773703:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4805 00000100:00100000:2.0:1761315799.773708:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dc64cfd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322589824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773744:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dc64cfd9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322589824:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (138us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315799.773745:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4806 00000100:00100000:2.0:1761315799.773759:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4805 00000100:00100000:3.0:1761315799.773761:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b66b3185 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322590208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773793:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322589824:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.773814:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322590208:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315799.773824:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b66b3185 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322590208:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (153us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315799.773832:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4806 00000100:00100000:1.0:1761315799.774026:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315799.774053:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af5d445a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322589632:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774079:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315799.774091:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322589888:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774109:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315799.774122:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322590272:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774140:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315799.774170:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590272 00000100:00100000:1.0:1761315799.774192:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1565 00000100:00100000:1.0:1761315799.774196:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322590272:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774226:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322590272:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (86us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315799.774232:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1565 00000100:00100000:1.0:1761315799.774251:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589888 00000100:00100000:1.0:1761315799.774273:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3001 00000100:00100000:1.0:1761315799.774277:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322589888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774307:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322589888:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315799.774315:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3001 00000100:00100000:1.0:1761315799.774359:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322589888:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315799.774371:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322590272:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315799.774490:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a07dadbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590080:12345-0@lo:8:kworker/u8:0.0 Request processed in 1082us (1357us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315799.774497:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2364 00000100:00100000:2.0:1761315799.774611:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589632 00000100:00100000:2.0:1761315799.774643:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1770 00000100:00100000:2.0:1761315799.774647:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322589632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315799.774675:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322589632:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (596us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315799.774680:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1770 00000100:00100000:3.0:1761315799.774790:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322589568 00000100:00100000:3.0:1761315799.774817:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1035 00000100:00100000:3.0:1761315799.774821:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322589568:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315799.774839:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315799.774864:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322589568:12345-0@lo:13:osp-pre-1-1.0 Request processed in 42us (839us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315799.774869:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1035 00000100:00080000:1.0:1761315799.775445:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315799.775450:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315799.775454:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322590080:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315799.775502:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322589568:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315799.775512:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af5d445a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322589632:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315800.795601:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315801.800758:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315801.800799:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315801.800828:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315801.800909:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566912 00000100:00100000:1.0:1761315801.800963:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566976 00000100:00100000:1.0:1761315801.800985:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4807 00000100:00100000:1.0:1761315801.800992:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000269e81e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307566912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315801.801064:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000269e81e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307566912:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 71us (313us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315801.801071:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4807 00000100:00100000:1.0:1761315801.801086:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4808 00000100:00100000:1.0:1761315801.801089:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000810216a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307566976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315801.801114:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000810216a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307566976:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315801.801119:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4808 00000100:00100000:0.0:1761315801.801292:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307566848 00000100:00100000:0.0:1761315801.801331:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1771 00000100:00100000:0.0:1761315801.801336:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307566848:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315801.801392:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307566848:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 55us (592us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315801.801397:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1771 00000100:00100000:2.0:1761315801.803263:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315801.803365:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567040 00000100:00100000:0.0:1761315801.803400:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2365 00000100:00100000:0.0:1761315801.803405:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307567040:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315801.803505:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315801.803558:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307567040:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 152us (299us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315801.803565:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2365 00010000:00010000:0.0:1761315801.819634:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315802.334159:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322590400:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315802.334230:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315802.334370:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590400 00000100:00100000:1.0:1761315802.334422:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3002 00000100:00100000:1.0:1761315802.334428:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322590400:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315802.334468:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315802.334525:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322590400:12345-0@lo:41:osp-pre-1-0.0 Request processed in 96us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315802.334531:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3002 00000100:00100000:3.0:1761315802.334646:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322590400:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315802.846139:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315803.868069:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315804.891607:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315804.891613:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315804.891617:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315804.891625:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4047 00000100:00080000:0.0:1761315804.891628:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315804.891634:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315804.891653:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d9777827 x1846869322590784/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315804.891666:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315804.891669:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315804.891671:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315804.891673:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4047 00000100:00080000:0.0:1761315804.891674:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315804.891677:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315804.891688:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000900f7fe8 x1846869322590848/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315804.891737:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9777827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322590784:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315804.891763:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315804.891792:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322590528:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.891801:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315804.891828:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000900f7fe8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322590848:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315804.891894:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322590464:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315804.892043:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322590656:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892075:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892095:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efccd99e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322590912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892118:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892140:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322591168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315804.892163:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892180:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315804.892185:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322590592:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892208:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892223:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322590976:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892245:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892279:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590528 00000100:00100000:1.0:1761315804.892283:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315804.892310:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590784 00000100:00100000:0.0:1761315804.892312:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590912 00000100:00100000:0.0:1761315804.892326:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590976 00000100:00100000:1.0:1761315804.892326:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590848 00000100:00100000:1.0:1761315804.892358:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2366 00000100:00100000:0.0:1761315804.892359:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4809 00000100:00100000:0.0:1761315804.892364:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a4f007af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322590528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.892364:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590784:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892418:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a4f007af pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322590528:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (343us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.892425:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4809 00000100:00100000:0.0:1761315804.892452:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4810 00000100:00100000:0.0:1761315804.892455:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000016417a05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322590912:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315804.892474:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315804.892492:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000016417a05 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322590912:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (374us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.892497:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4810 00000100:00100000:1.0:1761315804.892501:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590784:12345-0@lo:8:kworker/u8:0.0 Request processed in 137us (701us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315804.892506:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2366 00000100:00100000:0.0:1761315804.892507:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4811 00000100:00100000:0.0:1761315804.892510:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf4ea074 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322590976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.892516:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2367 00000100:00100000:1.0:1761315804.892518:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590848:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892531:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf4ea074 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322590976:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (287us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.892536:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4811 00000100:00100000:0.0:1761315804.892552:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590592 00000100:00100000:0.0:1761315804.892571:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3003 00000100:00100000:0.0:1761315804.892574:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001999cdc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322590592:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315804.892594:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315804.892613:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001999cdc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322590592:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (405us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315804.892613:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322590848:12345-0@lo:8:kworker/u8:0.0 Request processed in 94us (330us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315804.892619:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3003 00000100:00100000:1.0:1761315804.892619:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2367 00000100:00080000:1.0:1761315804.892651:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761315804.892652:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322590592:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315804.892654:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315804.892658:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9777827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322590784:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892660:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322590976:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315804.892664:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315804.892666:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315804.892667:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322590528:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.892668:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000900f7fe8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322590848:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892674:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efccd99e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322590912:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.892708:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591168 00000100:00100000:0.0:1761315804.892744:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1036 00000100:00100000:0.0:1761315804.892748:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eebe36d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322591168:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315804.892767:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315804.892799:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eebe36d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322591168:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (636us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.892804:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1036 00000100:00100000:0.0:1761315804.892819:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c88b345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322591232:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315804.892835:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ca7f87b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322591168:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315804.892840:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315804.892863:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591232 00000100:00100000:0.0:1761315804.892881:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3004 00000100:00100000:0.0:1761315804.892884:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efccd99e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322591232:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315804.892903:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315804.892942:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efccd99e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322591232:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (101us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.892948:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3004 00000100:00100000:1.0:1761315804.892986:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c88b345 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322591232:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315804.893181:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315804.893210:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322590720:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315804.893232:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315804.893253:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322591104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.893260:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590464 00000100:00100000:2.0:1761315804.893273:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315804.893283:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1772 00000100:00100000:1.0:1761315804.893286:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322590464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315804.893307:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591104 00000100:00100000:1.0:1761315804.893317:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322590464:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (137us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315804.893324:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1772 00000100:00100000:2.0:1761315804.893328:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1566 00000100:00100000:2.0:1761315804.893331:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322591104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315804.893374:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322591104:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (101us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315804.893380:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1566 00000100:00100000:2.0:1761315804.893415:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322590464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315804.893424:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322591104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.893575:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590720 00000100:00100000:0.0:1761315804.893606:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3005 00000100:00100000:0.0:1761315804.893610:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322590720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315804.893650:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322590720:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (417us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315804.893656:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3005 00000100:00100000:2.0:1761315804.893741:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322590720:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315804.893814:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315804.893838:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e83bd387 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322591040:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315804.893855:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315804.893911:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591040 00000100:00100000:2.0:1761315804.893936:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1567 00000100:00100000:2.0:1761315804.893939:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322591040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315804.893973:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322591040:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (118us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315804.893978:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1567 00000100:00100000:0.0:1761315804.894029:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e83bd387 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322591040:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.894543:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322590656 00000100:00100000:1.0:1761315804.894581:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4812 00000100:00100000:1.0:1761315804.894585:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322590656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315804.894630:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008135f598 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322590656:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (816us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315804.894636:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4812 00000100:00100000:0.0:1761315804.894703:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322590656:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315805.916371:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315806.924140:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315806.924192:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315806.924236:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315806.924240:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567168 00000100:00100000:2.0:1761315806.924293:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567232 00000100:00100000:1.0:1761315806.924312:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4813 00000100:00100000:1.0:1761315806.924320:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307567168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315806.924330:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4814 00000100:00100000:2.0:1761315806.924336:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307567232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315806.924430:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c88b345 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307567168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 101us (288us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315806.924439:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4813 00000100:00100000:2.0:1761315806.924677:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fb9fba07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307567232:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 339us (441us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315806.924685:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4814 00000100:00100000:0.0:1761315806.924715:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567104 00000100:00100000:0.0:1761315806.924764:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1773 00000100:00100000:0.0:1761315806.924768:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307567104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315806.924796:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307567104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 27us (605us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315806.924802:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1773 00000100:00100000:2.0:1761315806.927587:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315806.927676:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567296 00000100:00100000:3.0:1761315806.927724:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2368 00000100:00100000:3.0:1761315806.927730:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307567296:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315806.927821:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315806.928149:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307567296:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 417us (567us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315806.928157:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2368 00010000:00010000:0.0:1761315806.939579:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315807.451633:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322591296:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315807.451704:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315807.452017:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591296 00000100:00100000:1.0:1761315807.452212:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3006 00000100:00100000:1.0:1761315807.452218:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322591296:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315807.452294:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315807.453495:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322591296:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1273us (1788us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315807.453507:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3006 00000100:00100000:0.0:1761315807.453568:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322591296:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315807.963696:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315808.987547:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315810.011614:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322591360:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315810.011683:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.011795:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591360 00000100:00100000:1.0:1761315810.011844:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3007 00000100:00100000:1.0:1761315810.011850:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322591360:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315810.011884:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00010000:00010000:0.0:1761315810.011917:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315810.011935:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322591360:12345-0@lo:41:osp-pre-0-1.0 Request processed in 84us (252us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315810.011958:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3007 00000100:00100000:0.0:1761315810.011983:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322591360:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315810.012094:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322591552:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315810.012096:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315810.012108:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315810.012111:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315810.012117:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4052 00000100:00080000:2.0:1761315810.012120:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315810.012124:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315810.012132:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315810.012145:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000071ef691f x1846869322591744/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315810.012154:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:2.0:1761315810.012161:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315810.012161:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322591616:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315810.012167:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315810.012170:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315810.012174:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4052 00000100:00080000:2.0:1761315810.012176:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315810.012179:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315810.012189:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315810.012192:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f0419e0e x1846869322591808/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315810.012233:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591552 00000100:00100000:1.0:1761315810.012251:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091ae791a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322591488:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315810.012261:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591616 00000100:00100000:3.0:1761315810.012284:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4815 00000100:00100000:1.0:1761315810.012293:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315810.012293:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092b2ea3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322591552:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.012311:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322591872:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.012335:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.012366:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315810.012371:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ebc426e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322591424:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315810.012393:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315810.012395:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315810.012417:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322591936:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.012423:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322591680:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315810.012457:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.012458:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.012688:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591488 00000100:00100000:1.0:1761315810.012715:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1774 00000100:00100000:1.0:1761315810.012719:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322591488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.012751:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322591488:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (464us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315810.012757:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1774 00000100:00100000:1.0:1761315810.012783:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591872 00000100:00100000:1.0:1761315810.012799:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591936 00000100:00100000:1.0:1761315810.012809:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4816 00000100:00100000:1.0:1761315810.012812:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bceeb641 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322591616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315810.012853:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4817 00000100:00100000:0.0:1761315810.012859:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322591872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315810.012897:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e351e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322591872:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (561us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315810.012903:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4817 00000100:00100000:1.0:1761315810.012961:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bceeb641 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322591616:12345-0@lo:400:kworker/u8:2.0 Request processed in 147us (772us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315810.012965:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092b2ea3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322591552:12345-0@lo:400:kworker/u8:2.0 Request processed in 674us (832us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315810.012972:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4816 00000100:00100000:3.0:1761315810.012978:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4815 00000100:00100000:1.0:1761315810.012992:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4818 00000100:00100000:1.0:1761315810.012998:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e83bd387 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322591936:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315810.013033:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322591616:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315810.013089:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592000:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315810.013138:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591424 00000100:00100000:2.0:1761315810.013164:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1037 00000100:00100000:2.0:1761315810.013167:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322591424:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315810.013187:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315810.013477:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e83bd387 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322591936:12345-0@lo:400:kworker/u8:2.0 Request processed in 479us (1022us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315810.013484:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4818 00000100:00100000:1.0:1761315810.013515:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091ae791a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322591488:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013526:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d48b7a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322591872:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013549:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591680 00000100:00100000:1.0:1761315810.013573:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3008 00000100:00100000:1.0:1761315810.013577:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322591680:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013616:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322591680:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (1160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315810.013625:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3008 00000100:00100000:1.0:1761315810.013647:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322591680:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013687:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322591744:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013720:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.013736:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322591808:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013753:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315810.013776:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591744 00000100:00100000:1.0:1761315810.013793:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322591808 00000100:00100000:3.0:1761315810.013793:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322591552:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013805:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2369 00000100:00100000:1.0:1761315810.013808:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322591744:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315810.013891:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315810.013905:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2370 00000100:00100000:3.0:1761315810.013911:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322591808:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315810.013921:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322591744:12345-0@lo:8:kworker/u8:2.0 Request processed in 110us (200us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315810.013926:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2369 00000100:00080000:1.0:1761315810.013960:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315810.013964:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315810.013967:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322591744:0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315810.013989:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315810.014948:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322591808:12345-0@lo:8:kworker/u8:2.0 Request processed in 1034us (1191us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315810.014957:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2370 00000100:00100000:0.0:1761315810.015104:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315810.015112:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322591424:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1941us (2716us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315810.015121:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1037 00000100:00100000:0.0:1761315810.015141:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322591936:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315810.015169:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ebc426e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322591424:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315810.015180:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592000 00000100:00100000:0.0:1761315810.015212:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1568 00000100:00100000:0.0:1761315810.015218:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315810.015270:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592000:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (167us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315810.015275:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1568 00000100:00100000:2.0:1761315810.015315:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000178f802 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592000:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315810.015501:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315810.015505:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315810.015509:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0419e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322591808:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315811.035571:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315812.040633:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315812.040690:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315812.040729:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315812.040782:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567424 00000100:00100000:3.0:1761315812.040796:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567488 00000100:00100000:3.0:1761315812.040835:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4819 00000100:00100000:1.0:1761315812.040842:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4820 00000100:00100000:3.0:1761315812.040844:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307567488:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315812.040845:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000178f802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307567424:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315812.040910:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000178f802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307567424:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (286us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315812.040918:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4820 00000100:00100000:0.0:1761315812.040990:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567360 00000100:00100000:0.0:1761315812.041029:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1775 00000100:00100000:0.0:1761315812.041033:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307567360:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315812.041082:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0419e0e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307567488:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 233us (349us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315812.041089:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4819 00000100:00100000:1.0:1761315812.042916:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315812.043064:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307567360:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 2027us (2372us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315812.043078:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1775 00000100:00100000:3.0:1761315812.043793:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567552 00000100:00100000:3.0:1761315812.043835:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2371 00000100:00100000:3.0:1761315812.043841:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307567552:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315812.043936:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315812.044003:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307567552:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 161us (1094us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315812.044010:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2371 00010000:00010000:0.0:1761315812.059540:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315812.571875:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315812.571898:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322592064:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315812.571968:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315812.572058:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592064 00000100:00100000:0.0:1761315812.572102:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3009 00000100:00100000:0.0:1761315812.572106:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322592064:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315812.572134:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315812.572185:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322592064:12345-0@lo:41:osp-pre-1-0.0 Request processed in 77us (218us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315812.572192:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3009 00000100:00100000:3.0:1761315812.572263:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322592064:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315813.083566:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315814.107525:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315815.131510:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315815.132236:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592128:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315815.132286:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315815.132292:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315815.132296:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315815.132302:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4057 00000100:00080000:0.0:1761315815.132305:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315815.132312:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315815.132314:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132337:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a8f7f27b x1846869322592512/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315815.132388:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315815.132388:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf40c82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322592384:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315815.132395:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315815.132397:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315815.132402:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4057 00000100:00080000:0.0:1761315815.132405:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315815.132408:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315815.132422:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132423:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000087e39f9c x1846869322592576/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315815.132441:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315815.132450:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592320:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132475:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132498:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322592256:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132502:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592768:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132538:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315815.132543:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132559:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322592640:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132564:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315815.132572:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322592704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132588:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132620:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:2.0:1761315815.132627:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592128 00000100:00100000:0.0:1761315815.132628:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322592192:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132661:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315815.132666:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592320 00000100:00100000:0.0:1761315815.132676:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592448:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132689:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3010 00000100:00100000:2.0:1761315815.132696:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322592128:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315815.132703:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315815.132712:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592832:0@lo:400:kworker/u8:2.0 00000004:00000024:2.0:1761315815.132726:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315815.132735:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315815.132771:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322592128:12345-0@lo:41:osp-pre-0-1.0 Request processed in 74us (461us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315815.132778:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3010 00000100:00100000:0.0:1761315815.132793:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592384 00000100:00100000:2.0:1761315815.132798:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592448 00000100:00100000:2.0:1761315815.132817:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3011 00000100:00100000:2.0:1761315815.132821:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322592320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132824:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592256 00000100:00100000:0.0:1761315815.132842:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592640 00000100:00100000:2.0:1761315815.132854:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322592320:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (378us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.132860:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4821 00000100:00100000:2.0:1761315815.132864:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3011 00000100:00100000:0.0:1761315815.132867:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322592384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132880:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3012 00000100:00100000:2.0:1761315815.132885:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af5d445a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132905:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d48b7a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322592384:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (484us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.132913:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4821 00000100:00100000:0.0:1761315815.132930:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4822 00000100:00100000:2.0:1761315815.132930:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af5d445a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592448:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (226us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.132936:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000900f7fe8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322592256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.132942:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3012 00000100:00100000:0.0:1761315815.132971:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000900f7fe8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322592256:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (433us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.132977:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4822 00000100:00100000:2.0:1761315815.132985:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592448:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.132991:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4823 00000100:00100000:0.0:1761315815.132995:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9777827 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322592640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.133009:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322592256:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.133022:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9777827 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322592640:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (434us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315815.133027:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322592640:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.133029:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4823 00000100:00100000:0.0:1761315815.133066:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592768 00000100:00100000:0.0:1761315815.133084:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592832 00000100:00100000:0.0:1761315815.133096:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1569 00000100:00100000:0.0:1761315815.133098:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322592768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.133125:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322592768:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (585us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.133131:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1569 00000100:00100000:0.0:1761315815.133141:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1570 00000100:00100000:0.0:1761315815.133144:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.133172:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322592832:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (437us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.133176:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1570 00000100:00100000:3.0:1761315815.133252:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322592896:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315815.133281:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315815.133377:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322592832:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315815.133379:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592896 00000100:00100000:0.0:1761315815.133409:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1038 00000100:00100000:0.0:1761315815.133413:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322592896:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315815.133430:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315815.133473:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322592896:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (192us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315815.133479:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1038 00000100:00100000:1.0:1761315815.133548:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592192 00000100:00100000:1.0:1761315815.133584:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1776 00000100:00100000:1.0:1761315815.133588:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322592192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315815.133625:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322592896:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315815.133630:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322592192:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (967us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315815.133636:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1776 00000100:00100000:3.0:1761315815.133639:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322592192:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315815.133687:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322592512:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315815.133708:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315815.133720:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322592576:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315815.133733:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315815.133751:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592512 00000100:00100000:3.0:1761315815.133777:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592576 00000100:00100000:3.0:1761315815.133788:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2372 00000100:00100000:3.0:1761315815.133791:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322592512:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315815.133904:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315815.133928:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322592512:12345-0@lo:8:kworker/u8:2.0 Request processed in 135us (220us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315815.133932:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2372 00000100:00100000:3.0:1761315815.133941:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2373 00000100:00100000:3.0:1761315815.133944:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322592576:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315815.134010:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315815.134025:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322592576:12345-0@lo:8:kworker/u8:2.0 Request processed in 80us (291us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315815.134029:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2373 00000100:00080000:3.0:1761315815.134050:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315815.134053:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315815.134056:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322592512:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315815.134061:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315815.134063:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315815.134065:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322592576:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315815.140468:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315815.140545:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.140560:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592320:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.140571:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf40c82 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322592384:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315815.140589:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322592128:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315815.141261:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592704 00000100:00100000:3.0:1761315815.141326:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4824 00000100:00100000:3.0:1761315815.141333:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322592704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315815.141460:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322592704:12345-0@lo:400:kworker/u8:2.0 Request processed in 126us (1000us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315815.141467:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4824 00000100:00100000:2.0:1761315815.141554:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322592704:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315816.155607:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315817.163319:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315817.163431:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567616 00000100:00100000:0.0:1761315817.163504:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1777 00000100:00100000:0.0:1761315817.163511:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307567616:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315817.163596:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307567616:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 84us (388us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315817.163604:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1777 00000100:00100000:3.0:1761315817.164185:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315817.164236:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315817.164273:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567680 00000100:00100000:3.0:1761315817.164288:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315817.164333:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567744 00000100:00100000:3.0:1761315817.164387:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4825 00000100:00100000:3.0:1761315817.164398:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000620e4a68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307567680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315817.164418:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4826 00000100:00100000:2.0:1761315817.164425:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a421fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307567744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315817.164513:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567808 00000100:00100000:3.0:1761315817.164536:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000620e4a68 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307567680:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 138us (1088us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315817.164546:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2374 00000100:00100000:0.0:1761315817.164560:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307567808:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761315817.164569:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4825 00010000:02020000:0.0:1761315817.164654:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315817.164911:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a421fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307567744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 485us (675us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315817.164919:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4826 00000100:00100000:0.0:1761315817.164982:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307567808:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 425us (694us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315817.164989:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2374 00010000:00010000:2.0:1761315817.183666:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315817.691590:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315817.691613:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322592960:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315817.691681:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315817.691776:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322592960 00000100:00100000:3.0:1761315817.691829:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3013 00000100:00100000:3.0:1761315817.691835:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322592960:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315817.691869:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315817.691917:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322592960:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (237us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315817.691924:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3013 00000100:00100000:2.0:1761315817.692001:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322592960:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315818.214551:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315819.240914:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315820.251557:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315820.251573:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593024:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315820.251633:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315820.252325:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315820.252386:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315820.252394:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315820.252400:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4062 00000100:00080000:1.0:1761315820.252403:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315820.252409:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315820.252434:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a5918643 x1846869322593408/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315820.252449:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315820.252453:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315820.252454:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315820.252457:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4062 00000100:00080000:1.0:1761315820.252459:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315820.252462:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315820.252472:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bca5f7bb x1846869322593472/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315820.252519:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322593088:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252560:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.252584:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322593344:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252605:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.252620:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322593728:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252640:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.252661:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315820.252665:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322593152:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252689:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.252707:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322593536:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252731:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.252760:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593728 00000100:00100000:1.0:1761315820.252803:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1571 00000100:00100000:1.0:1761315820.252810:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322593728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.252860:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322593728:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.252866:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1571 00000100:00100000:0.0:1761315820.252872:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593024 00000100:00100000:1.0:1761315820.252892:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593152 00000100:00100000:0.0:1761315820.252908:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1039 00000100:00100000:0.0:1761315820.252913:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322593024:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315820.252925:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593536 00002000:00000024:0.0:1761315820.252938:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315820.252949:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4827 00000100:00100000:1.0:1761315820.252955:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061659632 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322593152:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.252974:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322593024:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (1341us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315820.252981:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1039 00000100:00100000:1.0:1761315820.252999:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061659632 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322593152:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.253007:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4827 00000100:00100000:0.0:1761315820.253023:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4828 00000100:00100000:0.0:1761315820.253028:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322593536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.253053:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058f03816 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322593536:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315820.253099:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4828 00000100:00100000:0.0:1761315820.253127:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593088 00000100:00100000:1.0:1761315820.253139:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000890fed6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322593152:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.253153:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322593536:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.253156:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1778 00000100:00100000:0.0:1761315820.253163:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322593088:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.253168:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322593728:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.253199:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dce6b44d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322593088:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.253200:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322593088:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (638us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315820.253210:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1778 00000100:00100000:1.0:1761315820.253230:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593344 00000100:00100000:0.0:1761315820.253258:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322593408:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315820.253260:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3014 00000100:00100000:1.0:1761315820.253267:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322593344:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.253291:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.253307:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322593344:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (701us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.253314:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3014 00000100:00100000:1.0:1761315820.253367:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322593344:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.253487:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593024:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315820.253505:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091ae791a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593216:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.253525:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315820.253537:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322593600:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.253553:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315820.253565:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315820.253569:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322593280:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.253584:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315820.253594:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5acab3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593664:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315820.253912:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315820.254000:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593600 00000100:00100000:2.0:1761315820.254032:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593280 00000100:00100000:2.0:1761315820.254058:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4829 00000100:00100000:2.0:1761315820.254062:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322593600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.254103:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322593792:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315820.255289:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.255379:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593408 00000100:00100000:1.0:1761315820.255405:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2375 00000100:00100000:1.0:1761315820.255409:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322593408:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315820.255496:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315820.255527:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322593408:12345-0@lo:8:kworker/u8:0.0 Request processed in 118us (2238us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315820.255538:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2375 00000100:00100000:1.0:1761315820.255575:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4830 00000100:00100000:1.0:1761315820.255581:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322593280:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.255614:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011c822db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322593280:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (2029us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.255620:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4830 00000100:00100000:1.0:1761315820.255646:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593216 00000100:00100000:1.0:1761315820.255669:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593792 00000100:00100000:1.0:1761315820.255680:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3015 00000100:00100000:1.0:1761315820.255683:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322593216:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315820.255713:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322593216:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (2188us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.255720:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3015 00000100:00100000:1.0:1761315820.255878:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3016 00000100:00100000:1.0:1761315820.255882:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322593792:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315820.255908:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315820.255944:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322593792:12345-0@lo:41:osp-pre-0-1.0 Request processed in 61us (660us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315820.255951:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3016 00000100:00100000:1.0:1761315820.255987:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce6ae3b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322593792:0@lo:41:osp-pre-0-1.0 00000100:00080000:0.0:1761315820.257878:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315820.257886:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315820.257892:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322593408:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315820.257902:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322593472:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315820.257945:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315820.258390:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593472 00000100:00100000:1.0:1761315820.258429:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2376 00000100:00100000:1.0:1761315820.258433:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322593472:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315820.258524:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315820.258566:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322593472:12345-0@lo:8:kworker/u8:0.0 Request processed in 131us (621us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315820.258573:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2376 00000100:00080000:0.0:1761315820.260002:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315820.260010:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315820.260015:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322593472:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315820.260174:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315820.260186:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006e05707 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322593600:12345-0@lo:400:kworker/u8:0.0 Request processed in 6121us (6630us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315820.260199:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4829 00000100:00100000:3.0:1761315820.260220:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091ae791a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593216:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.260234:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322593600:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315820.260259:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322593280:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.260605:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593664 00000100:00100000:0.0:1761315820.260646:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1572 00000100:00100000:0.0:1761315820.260651:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322593664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315820.260713:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322593664:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (542us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315820.260720:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1572 00000100:00100000:3.0:1761315820.260821:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5acab3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322593664:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315821.275775:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315822.280259:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315822.280303:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315822.280340:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315822.280439:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315822.280515:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567872 00000100:00100000:0.0:1761315822.280565:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1779 00000100:00100000:0.0:1761315822.280571:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307567872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315822.280628:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307567872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (376us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315822.280634:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1779 00000100:00100000:3.0:1761315822.282089:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307567936 00000100:00100000:3.0:1761315822.282143:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568000 00000100:00100000:3.0:1761315822.282168:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4831 00000100:00100000:3.0:1761315822.282174:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307567936:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315822.282224:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307567936:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (1920us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315822.282230:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4831 00000100:00100000:3.0:1761315822.282243:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4832 00000100:00100000:3.0:1761315822.282246:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307568000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315822.282261:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307568000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 15us (1923us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315822.282266:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4832 00000100:00100000:3.0:1761315822.282283:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568064 00000100:00100000:3.0:1761315822.282297:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2377 00000100:00100000:3.0:1761315822.282301:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c14c604d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307568064:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315822.282433:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315822.282457:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c14c604d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307568064:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 155us (2020us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315822.282463:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2377 00010000:00010000:3.0:1761315822.301780:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315822.812974:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315822.812994:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322593856:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315822.813055:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315822.813184:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593856 00000100:00100000:3.0:1761315822.813227:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3017 00000100:00100000:3.0:1761315822.813232:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322593856:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315822.813266:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315822.813316:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322593856:12345-0@lo:41:osp-pre-1-0.0 Request processed in 75us (253us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315822.813322:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3017 00000100:00100000:2.0:1761315822.813592:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf9b4b30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322593856:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315823.323695:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315824.347720:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315825.371627:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322593920:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315825.371646:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315825.371651:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315825.371655:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315825.371660:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1761315825.371662:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315825.371668:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315825.371687:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060d3b1bb x1846869322594240/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315825.371700:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315825.371703:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315825.371705:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315825.371708:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4067 00000100:00080000:1.0:1761315825.371710:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315825.371713:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315825.371723:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bca5f7bb x1846869322594304/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315825.371770:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322594048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.371770:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.371795:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d5acab3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594176:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.371806:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.371828:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315825.371833:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322594432:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.371839:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594560:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.371857:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.371858:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.371870:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315825.371871:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315825.371875:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322593984:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.371875:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322594112:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.371892:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.371905:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322594496:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.371908:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594048 00000100:00100000:3.0:1761315825.371922:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315825.371946:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594176 00000100:00100000:1.0:1761315825.371967:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3018 00000100:00100000:1.0:1761315825.371972:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322594048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372019:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322594048:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (212us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315825.372024:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3019 00000100:00100000:1.0:1761315825.372027:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3018 00000100:00100000:3.0:1761315825.372028:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594176:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372050:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594432 00000100:00100000:3.0:1761315825.372062:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594176:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (234us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315825.372068:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594112 00000100:00100000:3.0:1761315825.372068:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3019 00000100:00100000:1.0:1761315825.372084:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4833 00000100:00100000:3.0:1761315825.372085:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594560 00000100:00100000:1.0:1761315825.372087:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322594432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372114:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594496 00000100:00100000:1.0:1761315825.372128:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ae791a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322594432:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (270us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315825.372132:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1573 00000100:00100000:1.0:1761315825.372134:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4833 00000100:00100000:3.0:1761315825.372135:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594560:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372143:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4834 00000100:00100000:1.0:1761315825.372146:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322594112:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372161:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594560:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (303us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315825.372164:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca7f87b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322594112:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (273us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315825.372167:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1573 00000100:00100000:1.0:1761315825.372169:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4834 00000100:00100000:3.0:1761315825.372176:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1574 00000100:00100000:3.0:1761315825.372179:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322594496:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372188:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315825.372201:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322594368:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372201:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322594496:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (279us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315825.372206:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1574 00000100:00100000:1.0:1761315825.372219:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315825.372241:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322594048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372241:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322594112:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372250:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322594432:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372250:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322594496:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372257:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d5acab3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594176:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.372263:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000500f063b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594560:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315825.372273:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593984 00000100:00100000:1.0:1761315825.374546:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594368 00000100:00100000:1.0:1761315825.374853:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4835 00000100:00100000:1.0:1761315825.374861:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dce6b44d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322593984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315825.374887:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4836 00000100:00100000:2.0:1761315825.374893:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000890fed6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322594368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315825.375413:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594624:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315825.375454:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.375485:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315825.375492:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594688:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315825.375502:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315825.375514:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315825.375535:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594624 00000100:00100000:3.0:1761315825.375560:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322593920 00000100:00100000:0.0:1761315825.375564:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3020 00000100:00100000:0.0:1761315825.375568:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322594624:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315825.375582:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1780 00000100:00100000:3.0:1761315825.375585:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322593920:12345-0@lo:400:kworker/u8:2.0 00000004:00000024:0.0:1761315825.375590:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315825.375616:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322593920:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (3848us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315825.375619:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322594624:12345-0@lo:41:osp-pre-0-1.0 Request processed in 51us (166us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315825.375623:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1780 00000100:00100000:0.0:1761315825.375624:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3020 00000100:00100000:3.0:1761315825.375659:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322593920:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315825.375661:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594624:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315825.375687:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322594240:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315825.375711:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315825.375725:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322594304:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315825.375731:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000890fed6f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322594368:12345-0@lo:400:kworker/u8:2.0 Request processed in 837us (3512us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315825.375737:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4836 00000100:00100000:3.0:1761315825.375743:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315825.375758:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594688 00000100:00100000:3.0:1761315825.375765:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594240 00000100:00100000:3.0:1761315825.375778:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594304 00000100:00100000:2.0:1761315825.375780:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1040 00000100:00100000:2.0:1761315825.375783:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322594688:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315825.375789:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2378 00000100:00100000:3.0:1761315825.375791:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322594240:12345-0@lo:8:kworker/u8:2.0 00002000:00000024:2.0:1761315825.375802:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315825.375829:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322594688:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (315us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315825.375835:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1040 00000100:00100000:2.0:1761315825.375861:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594688:0@lo:13:osp-pre-1-1.0 00010000:02020000:3.0:1761315825.375880:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315825.375885:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2379 00000100:00100000:2.0:1761315825.375888:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322594304:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315825.375907:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322594240:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (196us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315825.375913:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2378 00000100:00080000:3.0:1761315825.375932:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315825.375936:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315825.375939:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060d3b1bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322594240:0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315825.375970:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315825.376011:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322594304:12345-0@lo:8:kworker/u8:2.0 Request processed in 121us (266us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315825.376017:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2379 00000100:00100000:1.0:1761315825.377530:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dce6b44d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322593984:12345-0@lo:400:kworker/u8:2.0 Request processed in 2663us (5335us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315825.377545:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4835 00000100:00080000:3.0:1761315825.377573:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315825.377579:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315825.377584:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bca5f7bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322594304:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315825.377650:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5fba45d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322593984:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315825.377663:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322594368:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315826.395670:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315827.407584:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315827.408013:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315827.408054:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315827.408096:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315827.408126:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568192 00000100:00100000:0.0:1761315827.408179:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568256 00000100:00100000:2.0:1761315827.408183:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568128 00000100:00100000:0.0:1761315827.408200:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4837 00000100:00100000:0.0:1761315827.408206:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf9b4b30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307568192:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315827.408209:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1781 00000100:00100000:2.0:1761315827.408212:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307568128:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315827.408271:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf9b4b30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307568192:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 65us (258us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315827.408288:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4837 00000100:00100000:0.0:1761315827.408305:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4838 00000100:00100000:1.0:1761315827.408306:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568320 00000100:00100000:0.0:1761315827.408309:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307568256:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315827.408326:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2380 00000100:00100000:1.0:1761315827.408330:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307568320:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761315827.408372:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307568256:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (319us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315827.408379:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4838 00000100:00100000:2.0:1761315827.408386:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307568128:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 172us (810us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315827.408392:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1781 00010000:02020000:1.0:1761315827.408772:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315827.410939:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307568320:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 2599us (2836us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315827.410955:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2380 00010000:00010000:0.0:1761315827.419858:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315827.931976:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315827.932501:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594752:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315827.932572:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315827.932830:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594752 00000100:00100000:3.0:1761315827.932886:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3021 00000100:00100000:3.0:1761315827.932892:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594752:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315827.932927:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315827.932981:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322594752:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (410us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315827.932988:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3021 00000100:00100000:1.0:1761315827.934017:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322594752:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315828.443808:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315829.467879:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315830.491500:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315830.491516:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322594816:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315830.491552:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315830.491558:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322594880:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315830.491558:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315830.491562:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315830.491568:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1761315830.491571:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315830.491577:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315830.491599:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1b571bc x1846869322595136/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315830.491615:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315830.491621:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315830.491622:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315830.491624:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315830.491629:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4072 00000100:00080000:2.0:1761315830.491631:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315830.491634:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315830.491643:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315830.491646:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d28148bd x1846869322595200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315830.491654:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315830.491663:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594944:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315830.491677:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322595008:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.491696:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315830.491715:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315830.491722:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322595264:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.491725:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594816 00000100:00100000:3.0:1761315830.491743:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322595392:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.491752:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315830.491776:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315830.491777:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1782 00000100:00100000:3.0:1761315830.491781:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315830.491784:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322595328:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.491789:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322594816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315830.491809:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315830.491817:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315830.491817:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322595072:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.491854:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594944 00000100:00100000:1.0:1761315830.491880:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3022 00000100:00100000:1.0:1761315830.491884:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322594944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315830.491964:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315830.491973:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322595520:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315830.492296:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322594816:12345-0@lo:400:kworker/u8:2.0 Request processed in 509us (654us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315830.492306:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1782 00000100:00100000:1.0:1761315830.492310:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322594944:12345-0@lo:400:kworker/u8:2.0 Request processed in 423us (614us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315830.492317:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3022 00000100:00100000:1.0:1761315830.492359:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322594880 00000100:00100000:2.0:1761315830.492380:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322595136:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315830.492382:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595008 00000100:00100000:1.0:1761315830.492399:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595264 00000100:00100000:1.0:1761315830.492411:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595328 00000100:00100000:2.0:1761315830.492414:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315830.492426:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4839 00000100:00100000:1.0:1761315830.492430:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bca5f7bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322594880:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.492434:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322595200:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315830.492468:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315830.492473:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bca5f7bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322594880:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (856us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315830.492480:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4839 00000100:00100000:1.0:1761315830.492496:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4840 00000100:00100000:1.0:1761315830.492501:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322595008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.492507:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4841 00000100:00100000:2.0:1761315830.492513:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322595264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.492529:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322595008:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (816us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315830.492535:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4840 00000100:00100000:2.0:1761315830.492550:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000500f063b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322595264:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (798us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315830.492554:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4842 00000100:00100000:1.0:1761315830.492562:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5acab3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322595328:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.492563:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4841 00000100:00100000:1.0:1761315830.492595:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5acab3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322595328:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (781us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315830.492608:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595392 00000100:00100000:2.0:1761315830.492627:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595136 00000100:00100000:2.0:1761315830.492638:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595200 00000100:00100000:2.0:1761315830.492648:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1575 00000100:00100000:2.0:1761315830.492651:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322595392:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.492675:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322595392:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (901us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315830.492680:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1575 00000100:00100000:2.0:1761315830.492691:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2381 00000100:00100000:2.0:1761315830.492694:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322595136:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315830.492784:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315830.492811:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322595136:12345-0@lo:8:kworker/u8:2.0 Request processed in 116us (401us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315830.492817:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2381 00000100:00100000:2.0:1761315830.492834:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2382 00000100:00100000:2.0:1761315830.492836:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322595200:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315830.492911:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315830.492929:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322595200:12345-0@lo:8:kworker/u8:2.0 Request processed in 92us (463us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315830.492949:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2382 00000100:00100000:1.0:1761315830.492960:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4842 00000100:00080000:2.0:1761315830.492977:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315830.492981:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315830.492985:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1b571bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322595136:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315830.492995:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315830.492998:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315830.493001:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d28148bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322595200:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315830.493014:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000023426d14 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322595328:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.493026:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322595264:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.493038:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322594944:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.493050:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322594880:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315830.493097:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315830.493105:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322595584:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315830.493129:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315830.493217:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315830.493242:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595520 00000100:00100000:0.0:1761315830.493273:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1041 00000100:00100000:0.0:1761315830.493276:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322595520:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315830.493294:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315830.493323:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322595520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315830.493328:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1041 00010000:00010000:0.0:1761315830.493389:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315830.493409:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322595520:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315830.494716:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315830.494790:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ded54f3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322595456:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.494822:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595584 00000100:00100000:3.0:1761315830.494827:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315830.494873:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595072 00000100:00100000:3.0:1761315830.494873:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322595392:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315830.494881:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595456 00000100:00100000:3.0:1761315830.494889:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322595008:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315830.494903:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322594816:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.494907:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3023 00000100:00100000:2.0:1761315830.494916:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322595584:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315830.494923:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1576 00000100:00100000:0.0:1761315830.494929:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322595456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315830.494943:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3024 00000004:00000024:2.0:1761315830.494948:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315830.494949:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322595072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315830.495922:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322595072:12345-0@lo:400:kworker/u8:2.0 Request processed in 969us (1207us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315830.495933:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322595456:12345-0@lo:400:kworker/u8:2.0 Request processed in 1002us (1104us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315830.495934:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3024 00000100:00100000:1.0:1761315830.496078:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322595072:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315830.496089:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1576 00000100:00100000:1.0:1761315830.496097:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ded54f3d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322595456:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315830.496146:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322595584:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1230us (3016us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315830.496154:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3023 00000100:00100000:0.0:1761315830.496199:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322595584:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315831.515528:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315832.526031:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315832.526095:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315832.526137:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315832.526175:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315832.526222:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568512 00000100:00100000:3.0:1761315832.526313:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568448 00000100:00100000:3.0:1761315832.526338:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4843 00000100:00100000:3.0:1761315832.526388:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093968913 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307568512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315832.526493:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093968913 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307568512:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 106us (356us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315832.526502:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4843 00000100:00100000:3.0:1761315832.526522:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4844 00000100:00100000:3.0:1761315832.526526:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307568448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315832.526552:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000421e79db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307568448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 25us (378us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315832.526558:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4844 00000100:00100000:2.0:1761315832.529409:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568576 00000100:00100000:2.0:1761315832.529478:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2383 00000100:00100000:2.0:1761315832.529487:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307568576:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315832.529581:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315832.529672:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568384 00000100:00100000:0.0:1761315832.529712:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1783 00000100:00100000:0.0:1761315832.529716:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307568384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315832.530079:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307568576:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 591us (4062us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315832.530088:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2383 00000100:00100000:0.0:1761315832.530170:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307568384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 451us (4074us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315832.530182:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1783 00010000:00010000:3.0:1761315832.539643:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315833.051750:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315833.051773:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322595648:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315833.051840:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315833.051929:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595648 00000100:00100000:3.0:1761315833.052322:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3025 00000100:00100000:3.0:1761315833.052328:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322595648:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315833.052381:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315833.052577:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322595648:12345-0@lo:41:osp-pre-1-0.0 Request processed in 248us (738us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315833.052585:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3025 00000100:00100000:2.0:1761315833.052662:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322595648:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315833.563529:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315834.587561:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761315835.611734:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315835.611740:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315835.611744:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315835.611751:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4078 00000100:00080000:0.0:1761315835.611754:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315835.611759:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315835.611776:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d287d70 x1846869322596032/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315835.611788:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315835.611791:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315835.611792:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315835.611795:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4078 00000100:00080000:0.0:1761315835.611796:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315835.611798:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315835.611805:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001b3dd283 x1846869322596096/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315835.611887:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b3a0ae9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322595776:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.611942:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315835.611965:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322596160:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.611982:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315835.612069:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8da4d9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322595904:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315835.612113:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322595840:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315835.612137:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315835.612152:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322596224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612255:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322595712:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612291:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315835.612310:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322595968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612339:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315835.612385:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322596352:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315835.612395:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315835.612412:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315835.612433:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595712 00000100:00100000:2.0:1761315835.612448:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322596032:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612474:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315835.612497:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322596096:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315835.612513:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1784 00000100:00100000:2.0:1761315835.612519:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315835.612526:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322595712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612559:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595840 00000100:00100000:2.0:1761315835.612596:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595968 00000100:00100000:2.0:1761315835.612628:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3026 00000100:00100000:2.0:1761315835.612637:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322595840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315835.612644:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322595712:12345-0@lo:400:kworker/u8:2.0 Request processed in 116us (347us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315835.612656:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1784 00000100:00100000:2.0:1761315835.612692:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322595840:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (553us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315835.612700:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3026 00000100:00100000:2.0:1761315835.612723:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3027 00000100:00100000:2.0:1761315835.612727:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322595968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315835.612744:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1993702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322595712:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612755:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322595968:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (416us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315835.612763:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3027 00000100:00100000:3.0:1761315835.612769:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322595968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612784:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596352 00000100:00100000:2.0:1761315835.612815:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596032 00000100:00100000:2.0:1761315835.612832:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596096 00000100:00100000:3.0:1761315835.612847:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1577 00000100:00100000:3.0:1761315835.612861:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322596352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.612862:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2384 00000100:00100000:2.0:1761315835.612875:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322596032:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315835.612923:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322596352:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (512us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315835.612932:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1577 00000100:00100000:3.0:1761315835.612951:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2385 00000100:00100000:3.0:1761315835.612961:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322596096:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315835.612970:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761315835.613049:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315835.613100:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322596096:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (583us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315835.613108:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2385 00000100:00100000:0.0:1761315835.613142:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f69efca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322596416:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315835.613154:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322596352:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.613183:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315835.613210:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315835.613228:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322596480:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315835.613249:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315835.613256:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315835.613263:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315835.613264:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322596096:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315835.613314:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596480 00000100:00100000:3.0:1761315835.613376:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596416 00000100:00100000:0.0:1761315835.613381:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1042 00000100:00100000:0.0:1761315835.613387:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322596480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315835.613417:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315835.613422:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3028 00000100:00100000:3.0:1761315835.613430:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322596416:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315835.613459:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322596480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (199us total) trans 0 rc 0/0 00000004:00000024:3.0:1761315835.613464:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315835.613469:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1042 00000100:00100000:0.0:1761315835.613508:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315835.613518:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322596416:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (335us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315835.613526:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3028 00000100:00100000:2.0:1761315835.614260:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322596032:12345-0@lo:8:kworker/u8:2.0 Request processed in 1383us (1783us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315835.614269:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2384 00000100:00100000:2.0:1761315835.614336:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322596480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315835.614371:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f69efca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322596416:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315835.614486:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595776 00000100:00100000:1.0:1761315835.614517:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596160 00000100:00100000:1.0:1761315835.614530:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322595904 00000100:00100000:1.0:1761315835.614545:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4845 00000100:00100000:1.0:1761315835.614549:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322595776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315835.614589:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d14b1c65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322595776:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (2648us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315835.614598:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4845 00000100:00100000:1.0:1761315835.614609:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4846 00000100:00100000:1.0:1761315835.614612:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322596160:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315835.614631:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fddf45ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322596160:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (2650us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315835.614636:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4846 00000100:00100000:1.0:1761315835.614644:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4847 00000100:00100000:1.0:1761315835.614647:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322595904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315835.614665:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000358802ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322595904:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (1161us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315835.614670:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4847 00000100:00100000:0.0:1761315835.614756:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8da4d9e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322595904:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.614764:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322596288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.614788:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315835.614869:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315835.614899:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000304ef22a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322595840:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.614948:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596224 00000100:00100000:0.0:1761315835.614975:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4848 00000100:00100000:0.0:1761315835.614978:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322596224:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315835.615066:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5fba45d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322596224:12345-0@lo:400:kworker/u8:2.0 Request processed in 86us (197us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315835.615072:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4848 00000100:00100000:1.0:1761315835.615135:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322596224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.615295:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b3a0ae9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322595776:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.615305:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322596160:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.615324:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596288 00000100:00100000:2.0:1761315835.615371:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1578 00000100:00100000:2.0:1761315835.615375:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322596288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315835.615415:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322596288:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (627us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315835.615421:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1578 00000100:00100000:0.0:1761315835.615483:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000078a92c3a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322596288:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315835.615600:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315835.615604:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315835.615608:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322596032:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315836.635608:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315837.640861:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315837.640914:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315837.640975:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568768 00000100:00100000:1.0:1761315837.641033:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4849 00000100:00100000:1.0:1761315837.641040:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307568768:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315837.641116:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000078a92c3a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307568768:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (203us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315837.641124:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4849 00000100:00100000:2.0:1761315837.641561:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568640 00000100:00100000:2.0:1761315837.641599:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1785 00000100:00100000:2.0:1761315837.641604:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307568640:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315837.641651:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307568640:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 47us (799us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315837.641658:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1785 00000100:00100000:0.0:1761315837.642999:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315837.644132:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568704 00000100:00100000:3.0:1761315837.644740:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4850 00000100:00100000:3.0:1761315837.644749:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307568704:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315837.645324:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000849700d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307568704:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 573us (2327us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315837.645333:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4850 00000100:00100000:0.0:1761315837.650058:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315837.650168:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568832 00000100:00100000:1.0:1761315837.650639:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2386 00000100:00100000:1.0:1761315837.650647:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008881ad51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307568832:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315837.650745:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315837.650814:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008881ad51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307568832:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 167us (764us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315837.650821:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2386 00010000:00010000:0.0:1761315837.659546:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315838.171585:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322596544:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315838.171658:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315838.171762:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596544 00000100:00100000:0.0:1761315838.171812:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3029 00000100:00100000:0.0:1761315838.171817:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322596544:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315838.171846:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315838.173231:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322596544:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1408us (1571us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315838.173243:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3029 00000100:00100000:1.0:1761315838.173380:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322596544:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315838.685017:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315839.707546:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315840.731666:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315840.732378:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322596672:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315840.732382:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315840.732387:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315840.732390:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315840.732394:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4083 00000100:00080000:1.0:1761315840.732397:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315840.732402:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315840.732422:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000afcc50ba x1846869322596928/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315840.732438:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315840.732441:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315840.732442:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315840.732445:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4083 00000100:00100000:3.0:1761315840.732445:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315840.732446:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315840.732449:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315840.732458:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000842f0f78 x1846869322596992/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315840.732478:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322597056:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732496:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5b00f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322596736:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732499:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315840.732522:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322596928:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732523:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315840.732538:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322597120:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732545:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315840.732554:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315840.732558:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322596992:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732575:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315840.732595:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596672 00000100:00100000:3.0:1761315840.732597:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596928 00000100:00100000:3.0:1761315840.732621:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2387 00000100:00100000:1.0:1761315840.732623:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597056 00000100:00100000:3.0:1761315840.732626:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322596928:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732636:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597120 00000100:00100000:1.0:1761315840.732649:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4851 00000100:00100000:1.0:1761315840.732652:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322596672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732710:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fcb6e1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322596672:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (266us total) trans 0 rc 0/0 00010000:02020000:3.0:1761315840.732711:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315840.732716:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4851 00000100:00100000:1.0:1761315840.732732:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4852 00000100:00100000:1.0:1761315840.732735:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322597056:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732738:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322596928:12345-0@lo:8:kworker/u8:0.0 Request processed in 111us (193us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315840.732743:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2387 00000100:00100000:3.0:1761315840.732753:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596992 00000100:00100000:1.0:1761315840.732757:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9d9bc94 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322597056:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315840.732762:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4852 00000100:00100000:3.0:1761315840.732767:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2388 00000100:00100000:3.0:1761315840.732770:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322596992:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732771:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4853 00000100:00100000:1.0:1761315840.732774:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322597120:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315840.732820:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9a1340f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322597120:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315840.732825:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4853 00010000:02020000:3.0:1761315840.732844:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315840.732866:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322596992:12345-0@lo:8:kworker/u8:0.0 Request processed in 95us (290us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315840.732871:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2388 00000100:00100000:1.0:1761315840.732880:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322597120:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732900:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322597056:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732908:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008cd9755b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322596672:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315840.732928:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315840.732932:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315840.732935:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322596928:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315840.732942:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315840.732945:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315840.732946:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000842f0f78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322596992:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315840.732975:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322596800:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.732976:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596736 00000100:00100000:3.0:1761315840.732997:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315840.733013:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3030 00000100:00100000:3.0:1761315840.733015:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596800 00000100:00100000:0.0:1761315840.733017:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322596736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733034:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4854 00000100:00100000:3.0:1761315840.733038:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322596800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733050:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322596736:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (527us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315840.733056:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3030 00000100:00100000:3.0:1761315840.733059:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c56478c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322596800:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (62us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315840.733064:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4854 00000100:00100000:3.0:1761315840.733077:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322597184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733079:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5b00f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322596736:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733098:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315840.733125:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597184 00000100:00100000:3.0:1761315840.733129:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322596800:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733137:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:3.0:1761315840.733142:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322596608:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733148:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1579 00000100:00100000:0.0:1761315840.733151:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322597184:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733166:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315840.733175:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322597184:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (78us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315840.733177:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003d02f4dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322596864:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733209:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315840.733210:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1579 00000100:00100000:3.0:1761315840.733221:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322597248:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733228:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596864 00000100:00100000:3.0:1761315840.733240:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315840.733247:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3031 00000100:00100000:0.0:1761315840.733250:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322596864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.733255:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322597184:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733551:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322596864:12345-0@lo:400:kworker/u8:0.0 Request processed in 298us (340us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315840.733557:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3031 00000100:00100000:0.0:1761315840.733574:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597248 00000100:00100000:0.0:1761315840.733593:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1580 00000100:00100000:0.0:1761315840.733596:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322597248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.733623:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322597248:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (383us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315840.733628:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1580 00000100:00100000:2.0:1761315840.734429:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322596608 00000100:00100000:0.0:1761315840.734453:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315840.734465:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322597312:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315840.734467:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1786 00000100:00100000:2.0:1761315840.734475:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322596608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315840.734508:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315840.734522:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322596608:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1354us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315840.734530:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1786 00000100:00100000:1.0:1761315840.734667:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597312 00000100:00100000:1.0:1761315840.734703:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3032 00000100:00100000:1.0:1761315840.734707:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322597312:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315840.734730:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315840.734781:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322597312:12345-0@lo:41:osp-pre-0-1.0 Request processed in 73us (274us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315840.734787:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3032 00000100:00100000:2.0:1761315840.734890:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322597376:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315840.734929:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315840.734965:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322597312:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315840.735326:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597376 00000100:00100000:1.0:1761315840.735380:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1043 00000100:00100000:1.0:1761315840.735384:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322597376:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315840.735403:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315840.735434:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322597376:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (505us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315840.735440:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1043 00000100:00100000:2.0:1761315840.735512:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322597376:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315840.739468:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322596608:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.739493:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003d02f4dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322596864:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315840.739507:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0b6ae48 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322597248:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315841.758479:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315842.760437:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315842.760483:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315842.760526:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315842.760561:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315842.760646:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568960 00000100:00100000:3.0:1761315842.760701:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569024 00000100:00100000:3.0:1761315842.760725:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4855 00000100:00100000:3.0:1761315842.760731:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307568960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315842.760804:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e962613 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307568960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (375us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315842.760812:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4855 00000100:00100000:3.0:1761315842.760829:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4856 00000100:00100000:3.0:1761315842.760833:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315842.760857:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c5ffead pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (297us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315842.760863:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4856 00000100:00100000:3.0:1761315842.760884:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307568896 00000100:00100000:3.0:1761315842.760906:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1787 00000100:00100000:3.0:1761315842.760910:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307568896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315842.760934:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307568896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (408us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315842.760940:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1787 00000100:00100000:2.0:1761315842.761509:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569088 00000100:00100000:2.0:1761315842.761547:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2389 00000100:00100000:2.0:1761315842.761552:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307569088:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315842.761641:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315842.762838:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307569088:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 1280us (2350us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315842.762849:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2389 00010000:00010000:0.0:1761315842.779525:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315843.292726:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315843.293515:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322597440:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315843.293585:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315843.293713:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597440 00000100:00100000:1.0:1761315843.293767:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3033 00000100:00100000:1.0:1761315843.293773:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a476cb8a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322597440:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315843.293807:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315843.294720:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322597440:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315843.294786:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a476cb8a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322597440:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1011us (1200us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315843.294794:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3033 00010000:00010000:1.0:1761315843.803821:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315844.827524:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315845.851538:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315845.851554:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e2af60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322597504:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315845.851584:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315845.851590:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315845.851594:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315845.851601:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322597568:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315845.851604:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4088 00000100:00080000:0.0:1761315845.851610:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315845.851616:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315845.851628:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315845.851637:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315845.851640:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000022eb81a x1846869322597824/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315845.851659:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315845.851661:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322597696:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.851661:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:0.0:1761315845.851665:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315845.851668:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315845.851670:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322597632:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315845.851674:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4088 00000100:00080000:0.0:1761315845.851677:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315845.851682:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315845.851687:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315845.851693:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315845.851698:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000046bac38f x1846869322597888/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315845.851698:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322597760:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.851724:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597568 00000100:00100000:2.0:1761315845.851743:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315845.851759:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315845.851767:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597696 00000100:00100000:2.0:1761315845.851776:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598144:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.851794:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315845.851794:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4857 00000100:00100000:3.0:1761315845.851800:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033e21383 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322597568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.851819:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315845.851823:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c6c8f7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598272:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315845.851843:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4858 00000100:00100000:1.0:1761315845.851844:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315845.851844:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033e21383 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322597568:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (208us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315845.851854:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322597696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.851855:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4857 00000100:00100000:1.0:1761315845.851860:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315845.851866:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322597952:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.851884:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597632 00000100:00100000:1.0:1761315845.851903:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315845.851911:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315845.851913:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597760 00000100:00100000:0.0:1761315845.851914:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000842f0f78 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322597696:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (226us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315845.851922:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003d02f4dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322598080:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.851923:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5b00f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322598208:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315845.851926:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4858 00000100:00100000:3.0:1761315845.851928:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598272 00000100:00100000:3.0:1761315845.851941:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3034 00000100:00100000:3.0:1761315845.851946:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322597632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.851955:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315845.851967:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597952 00000100:00100000:2.0:1761315845.851969:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315845.851979:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322597632:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315845.851987:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3034 00000100:00100000:1.0:1761315845.851990:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598016 00000100:00100000:0.0:1761315845.851998:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4859 00000100:00100000:3.0:1761315845.852004:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3035 00000100:00100000:0.0:1761315845.852008:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322597952:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.852013:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322597760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852018:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3036 00000100:00100000:1.0:1761315845.852023:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1044 00000100:00100000:2.0:1761315845.852027:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322598272:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315845.852031:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322598016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315845.852058:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000004:00000024:2.0:1761315845.852059:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315845.852064:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0b6ae48 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322597952:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (159us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315845.852065:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322597760:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315845.852078:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4859 00000100:00100000:3.0:1761315845.852079:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3035 00000100:00100000:1.0:1761315845.852088:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322598016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (296us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315845.852095:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1044 00000100:00100000:0.0:1761315845.852101:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598080 00000100:00100000:2.0:1761315845.852110:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322598272:12345-0@lo:41:osp-pre-0-1.0 Request processed in 83us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315845.852117:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3036 00000100:00100000:3.0:1761315845.852137:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4860 00000100:00100000:1.0:1761315845.852138:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598144 00000100:00100000:3.0:1761315845.852142:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071a6088f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322598080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852164:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598208 00000100:00100000:1.0:1761315845.852189:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1581 00000100:00100000:3.0:1761315845.852191:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071a6088f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322598080:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315845.852196:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322598144:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.852199:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4860 00000100:00100000:0.0:1761315845.852202:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322597952:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852206:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1582 00000100:00100000:2.0:1761315845.852214:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322598208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315845.852229:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c6c8f7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598272:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315845.852235:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322598144:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (419us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315845.852240:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322597632:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.852243:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1581 00000100:00100000:0.0:1761315845.852255:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315845.852259:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bbb442e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322597568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852263:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322598208:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (305us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315845.852272:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1582 00000100:00100000:1.0:1761315845.852276:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322597696:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.852290:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322597824:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315845.852308:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5b00f51 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322598208:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.852311:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597504 00000100:00100000:3.0:1761315845.852322:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315845.852328:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003d02f4dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322598080:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315845.852361:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1788 00000100:00100000:1.0:1761315845.852368:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322597504:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.852368:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322597888:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852374:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598144:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315845.852391:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322597760:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315845.852393:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315845.852398:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315845.852422:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597824 00000100:00100000:1.0:1761315845.852423:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322597504:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (796us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315845.852431:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1788 00000100:00100000:3.0:1761315845.852440:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322597888 00000100:00100000:0.0:1761315845.852452:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2390 00000100:00100000:0.0:1761315845.852458:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322597824:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315845.852468:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e2af60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322597504:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315845.852478:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2391 00000100:00100000:3.0:1761315845.852481:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322597888:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315845.852850:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315845.852901:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322597824:12345-0@lo:8:kworker/u8:2.0 Request processed in 443us (580us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315845.852908:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2390 00000100:00080000:2.0:1761315845.852967:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00010000:02020000:3.0:1761315845.852968:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1761315845.852974:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315845.852980:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322597824:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315845.853012:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322597888:12345-0@lo:8:kworker/u8:2.0 Request processed in 529us (615us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315845.853021:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2391 00000100:00080000:2.0:1761315845.853034:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315845.853038:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315845.853042:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322597888:0@lo:8:kworker/u8:2.0 00010000:00010000:1.0:1761315846.876525:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315847.883884:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315847.884178:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569344 00000100:00100000:3.0:1761315847.884230:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2392 00000100:00100000:3.0:1761315847.884238:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c6c8f7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307569344:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315847.884335:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315847.884443:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c6c8f7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307569344:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 204us (567us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315847.884452:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2392 00000100:00100000:3.0:1761315847.885500:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315847.886735:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569216 00000100:00100000:0.0:1761315847.886851:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4861 00000100:00100000:0.0:1761315847.886860:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbb442e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307569216:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315847.888516:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbb442e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307569216:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1652us (3015us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315847.888530:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4861 00000100:00100000:2.0:1761315847.889988:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315847.890034:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315847.890124:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569152 00000100:00100000:0.0:1761315847.890185:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1789 00000100:00100000:0.0:1761315847.890193:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307569152:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315847.890269:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307569152:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 76us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315847.890276:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1789 00000100:00100000:3.0:1761315847.890363:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569280 00000100:00100000:3.0:1761315847.890400:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4862 00000100:00100000:3.0:1761315847.890404:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec256aee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307569280:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315847.890435:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec256aee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307569280:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 30us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315847.890441:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4862 00010000:00010000:3.0:1761315847.901392:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315848.412148:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322598336:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315848.412219:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315848.412322:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598336 00000100:00100000:0.0:1761315848.412400:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3037 00000100:00100000:0.0:1761315848.412406:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322598336:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315848.412444:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315848.414210:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322598336:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1797us (1985us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315848.414224:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3037 00000100:00100000:2.0:1761315848.414703:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322598336:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315848.923926:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315849.948447:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315850.971560:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315850.971567:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315850.971570:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761315850.971572:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322598592:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315850.971579:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4093 00000100:00080000:3.0:1761315850.971583:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315850.971588:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00010000:00010000:0.0:1761315850.971591:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315850.971607:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000002e606f9 x1846869322598848/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315850.971641:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315850.971662:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322598848:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315850.971680:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315850.971688:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598528:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315850.971693:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598464:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315850.971697:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315850.971725:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315850.971735:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315850.971735:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315850.971739:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315850.971741:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315850.971746:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4093 00000100:00080000:3.0:1761315850.971747:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:1.0:1761315850.971750:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e2af60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322598784:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315850.971753:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315850.971758:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322598400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315850.971769:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1f55899 x1846869322598912/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315850.971785:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315850.971795:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315850.971806:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322598912:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315850.971814:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322598720:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.971824:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598592 00000100:00100000:0.0:1761315850.971833:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315850.971833:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315850.971852:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315850.971857:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598656:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315850.971877:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315850.971880:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598848 00000100:00100000:1.0:1761315850.971887:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598720 00000100:00100000:0.0:1761315850.971903:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598912 00000100:00100000:1.0:1761315850.971921:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4863 00000100:00100000:0.0:1761315850.971927:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2393 00000100:00100000:1.0:1761315850.971930:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322598592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.971935:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322598848:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315850.971973:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598464 00000100:00100000:1.0:1761315850.971983:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087ee706d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322598592:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.971991:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4863 00000100:00100000:2.0:1761315850.971999:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598784 00000100:00100000:1.0:1761315850.972011:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4864 00000100:00100000:2.0:1761315850.972015:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598656 00000100:00100000:1.0:1761315850.972018:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322598720:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315850.972023:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315850.972027:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3038 00000100:00100000:2.0:1761315850.972031:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322598464:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315850.972055:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315850.972061:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007aa8e5f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322598720:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (227us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315850.972068:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322598848:12345-0@lo:8:kworker/u8:2.0 Request processed in 131us (369us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315850.972073:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4864 00000100:00100000:0.0:1761315850.972079:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2393 00000100:00100000:2.0:1761315850.972083:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322598464:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315850.972089:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3038 00000100:00100000:0.0:1761315850.972095:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2394 00000100:00100000:0.0:1761315850.972100:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322598912:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315850.972102:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3039 00000100:00100000:2.0:1761315850.972106:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322598784:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.972108:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598528 00000100:00100000:2.0:1761315850.972149:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322598784:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (365us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.972153:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1045 00000100:00100000:2.0:1761315850.972160:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3039 00000100:00100000:1.0:1761315850.972161:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322598528:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315850.972175:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3040 00000100:00100000:2.0:1761315850.972180:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322598656:12345-0@lo:400:kworker/u8:2.0 00002000:00000024:1.0:1761315850.972183:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00010000:02020000:0.0:1761315850.972194:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315850.972528:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315850.972537:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322598976:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.972598:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322598528:12345-0@lo:13:osp-pre-1-1.0 Request processed in 437us (873us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.972606:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1045 00000100:00100000:1.0:1761315850.972633:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322599104:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.972658:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315850.972690:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598656:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.972701:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322598912:12345-0@lo:8:kworker/u8:2.0 Request processed in 599us (868us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315850.972707:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322598464:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315850.972711:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2394 00000100:00100000:1.0:1761315850.972723:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322598720:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.972729:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599104 00000100:00100000:1.0:1761315850.972748:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315850.972754:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322599040:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.972759:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1583 00000100:00100000:0.0:1761315850.972765:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322599104:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.972780:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315850.972809:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322599104:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.972811:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322599104:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.972823:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1583 00000100:00100000:0.0:1761315850.972857:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599040 00000100:00100000:1.0:1761315850.972876:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322598528:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315850.972886:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e2af60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322598784:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.972894:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b52f64ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322598592:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315850.972950:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315850.972953:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315850.972956:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322598912:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315850.972967:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315850.972969:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315850.972971:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322598848:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315850.973021:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598400 00000100:00100000:1.0:1761315850.973046:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1790 00000100:00100000:1.0:1761315850.973049:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322598400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.973083:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322598400:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (1290us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.973090:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1790 00000100:00100000:1.0:1761315850.973186:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002d5bc7ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322598400:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315850.973403:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322598656:12345-0@lo:400:kworker/u8:2.0 Request processed in 1221us (1524us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315850.973410:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3040 00000100:00100000:3.0:1761315850.973581:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315850.973685:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4865 00000100:00100000:0.0:1761315850.973691:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322599040:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315850.973728:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f03965b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322599040:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (949us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315850.973734:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4865 00000100:00100000:1.0:1761315850.973763:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322598976 00000100:00100000:1.0:1761315850.973794:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4866 00000100:00100000:1.0:1761315850.973797:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b3a0ae9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322598976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315850.975084:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f62317db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322598976:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315850.975153:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b3a0ae9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322598976:12345-0@lo:400:kworker/u8:2.0 Request processed in 1353us (1572us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315850.975160:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4866 00000100:00100000:2.0:1761315850.976757:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322599040:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315851.995505:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315853.003260:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315853.003313:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315853.003516:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569408 00000100:00100000:0.0:1761315853.003569:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1791 00000100:00100000:0.0:1761315853.003577:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307569408:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315853.003651:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307569408:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 74us (399us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315853.003660:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1791 00000100:00100000:2.0:1761315853.005732:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569536 00000100:00100000:2.0:1761315853.005804:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4867 00000100:00100000:2.0:1761315853.005811:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569536:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315853.005885:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f62317db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569536:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 74us (2572us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315853.005892:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4867 00000100:00100000:3.0:1761315853.007662:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315853.009329:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315853.009452:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569472 00000100:00100000:2.0:1761315853.009474:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569600 00000100:00100000:1.0:1761315853.009514:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4868 00000100:00100000:2.0:1761315853.009519:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2395 00000100:00100000:1.0:1761315853.009523:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307569472:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315853.009535:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307569600:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:1.0:1761315853.009603:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000304ef22a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307569472:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 81us (1945us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315853.009611:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4868 00010000:02020000:2.0:1761315853.009634:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315853.009666:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307569600:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 132us (344us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315853.009672:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2395 00010000:00010000:1.0:1761315853.022915:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315853.533159:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322599168:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761315853.533226:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315853.533280:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599168 00000100:00100000:3.0:1761315853.533323:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3041 00000100:00100000:3.0:1761315853.533330:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000765da965 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322599168:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315853.533397:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315853.533441:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000765da965 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322599168:12345-0@lo:41:osp-pre-1-0.0 Request processed in 109us (216us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315853.533449:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3041 00000100:00100000:3.0:1761315853.533515:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322599168:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315854.043536:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315855.067514:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315856.091609:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315856.091632:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322599232:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315856.091695:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315856.091746:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599232 00000100:00100000:1.0:1761315856.091786:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1046 00000100:00100000:1.0:1761315856.091792:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322599232:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315856.091823:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315856.091864:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322599232:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315856.091871:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1046 00000100:00100000:1.0:1761315856.091917:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322599232:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315856.092103:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315856.092114:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322599296:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315856.092146:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315856.092217:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599296 00000100:00100000:2.0:1761315856.092254:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3042 00000100:00100000:2.0:1761315856.092258:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322599296:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315856.092282:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315856.092318:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322599296:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315856.092326:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3042 00000100:00080000:0.0:1761315856.092369:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315856.092375:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315856.092378:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315856.092384:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4098 00000100:00080000:0.0:1761315856.092387:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315856.092388:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322599488:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761315856.092393:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315856.092412:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000157d6de5 x1846869322599616/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315856.092423:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315856.092425:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315856.092429:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315856.092430:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315856.092434:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4098 00000100:00080000:0.0:1761315856.092436:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315856.092439:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315856.092445:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315856.092451:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002694b4ae x1846869322599680/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315856.092454:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e0e11476 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322599424:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092488:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315856.092506:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322599872:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092531:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315856.092547:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315856.092554:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322599808:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315856.092556:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599872 00000100:00100000:2.0:1761315856.092578:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315856.092600:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1584 00000100:00100000:0.0:1761315856.092604:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322599872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092623:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599488 00000100:00100000:0.0:1761315856.092644:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322599872:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (112us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315856.092651:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1584 00000100:00100000:2.0:1761315856.092651:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599424 00000100:00100000:2.0:1761315856.092671:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599808 00000100:00100000:2.0:1761315856.092692:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4869 00000100:00100000:2.0:1761315856.092697:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322599488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315856.092703:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4870 00000100:00100000:0.0:1761315856.092708:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322599424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092736:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000061a066c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322599488:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (313us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315856.092744:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004580caf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322599424:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (257us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315856.092745:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4869 00000100:00100000:0.0:1761315856.092753:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4870 00000100:00100000:2.0:1761315856.092766:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4871 00000100:00100000:2.0:1761315856.092772:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ded54f3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322599808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092802:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ded54f3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322599808:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (225us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315856.092807:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4871 00000100:00100000:2.0:1761315856.092835:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322599808:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092843:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322599872:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092851:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e0e11476 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322599424:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.092858:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322599488:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315856.092949:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd6d9470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322599360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315856.092979:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315856.093004:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322599744:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315856.093024:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315856.093057:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322599296:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315856.093070:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315856.093086:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c28f1f32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322599552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315856.093108:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315856.093131:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599552 00000100:00100000:3.0:1761315856.093151:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3043 00000100:00100000:3.0:1761315856.093154:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322599552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315856.093186:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322599552:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (78us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315856.093191:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3043 00000100:00100000:3.0:1761315856.093213:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c28f1f32 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322599552:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.093276:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599360 00000100:00100000:2.0:1761315856.093308:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1792 00000100:00100000:2.0:1761315856.093312:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322599360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315856.093367:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322599360:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (387us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315856.093372:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1792 00000100:00100000:2.0:1761315856.093405:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd6d9470 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322599360:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315856.093509:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000157d6de5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322599616:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315856.093537:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315856.093561:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002694b4ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322599680:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315856.093580:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315856.093616:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599744 00000100:00100000:1.0:1761315856.093636:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4872 00000100:00100000:1.0:1761315856.093639:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1993702 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322599744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315856.093667:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1993702 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322599744:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315856.093673:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4872 00010000:00010000:1.0:1761315856.093688:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315856.093758:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599616 00000100:00100000:3.0:1761315856.093783:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599680 00000100:00100000:3.0:1761315856.093800:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2396 00000100:00100000:3.0:1761315856.093803:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322599616:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315856.093899:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315856.093931:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322599616:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (393us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315856.093937:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2396 00000100:00100000:3.0:1761315856.093948:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2397 00000100:00100000:3.0:1761315856.093951:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322599680:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315856.094027:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:0.0:1761315856.094033:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315856.094038:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315856.094043:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000157d6de5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322599616:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315856.094051:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322599680:12345-0@lo:8:kworker/u8:0.0 Request processed in 98us (469us total) trans 0 rc -19/-19 00000100:00080000:0.0:1761315856.094056:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315856.094060:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315856.094060:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2397 00000100:00100000:0.0:1761315856.094065:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002694b4ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322599680:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315856.094329:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322599744:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315857.115768:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315858.123751:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315858.129373:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569664 00000100:00100000:3.0:1761315858.129449:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1793 00000100:00100000:3.0:1761315858.129456:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307569664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315858.129537:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307569664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 81us (5794us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315858.129545:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1793 00000100:00100000:0.0:1761315858.129819:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315858.132233:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569728 00000100:00100000:1.0:1761315858.133215:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4873 00000100:00100000:1.0:1761315858.133225:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002694b4ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307569728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315858.133330:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002694b4ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307569728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 104us (3511us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315858.133365:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4873 00000100:00100000:0.0:1761315858.134163:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315858.134188:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315858.134266:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569792 00000100:00100000:1.0:1761315858.134309:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4874 00000100:00100000:1.0:1761315858.134314:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000157d6de5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315858.134378:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000157d6de5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307569792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 62us (217us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315858.134385:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4874 00000100:00100000:2.0:1761315858.134479:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569856 00000100:00100000:2.0:1761315858.134512:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2398 00000100:00100000:2.0:1761315858.134517:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307569856:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315858.134604:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315858.134642:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307569856:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 124us (453us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315858.134649:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2398 00010000:00010000:0.0:1761315858.141141:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315858.653396:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322599936:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315858.653473:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315858.658538:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322599936 00000100:00100000:3.0:1761315858.658622:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3044 00000100:00100000:3.0:1761315858.658629:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322599936:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315858.658686:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315858.658763:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322599936:12345-0@lo:41:osp-pre-1-0.0 Request processed in 134us (5299us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315858.658770:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3044 00000100:00100000:1.0:1761315858.659051:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322599936:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315859.163840:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315860.187515:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315861.211518:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315861.211676:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315861.211681:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315861.211684:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315861.211691:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4103 00000100:00080000:2.0:1761315861.211694:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315861.211701:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315861.211721:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c14c604d x1846869322600320/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315861.211740:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315861.211744:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315861.211745:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315861.211748:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4103 00000100:00080000:2.0:1761315861.211750:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315861.211752:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315861.211762:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aad6d994 x1846869322600384/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315861.211805:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322600128:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.211863:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315861.211891:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322600512:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.211915:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315861.211934:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315861.211940:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322600064:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.211962:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315861.211974:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322600448:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.211994:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315861.212025:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600128 00000100:00100000:2.0:1761315861.212080:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3045 00000100:00100000:2.0:1761315861.212086:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322600128:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.212132:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322600128:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (269us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315861.212138:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3045 00000100:00100000:2.0:1761315861.212174:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322600128:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212308:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322600000:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212370:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212386:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322600256:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212406:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212417:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322600640:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212434:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212445:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315861.212449:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a07dadbc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322600192:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212464:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212475:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322600576:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212489:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212506:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322600320:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212522:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212532:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322600384:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212546:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315861.212567:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600512 00000100:00100000:0.0:1761315861.212588:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600064 00000100:00100000:0.0:1761315861.212599:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600448 00000100:00100000:0.0:1761315861.212606:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600192 00000100:00100000:0.0:1761315861.212614:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4875 00000100:00100000:0.0:1761315861.212617:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322600512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212643:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000023426d14 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322600512:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (728us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212648:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4875 00000100:00100000:0.0:1761315861.212658:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1761315861.212660:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322600064:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212676:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d28148bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322600064:12345-0@lo:400:kworker/u8:2.0 Request processed in 15us (715us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212680:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4876 00000100:00100000:0.0:1761315861.212687:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4877 00000100:00100000:0.0:1761315861.212689:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1b571bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322600448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212704:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1b571bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322600448:12345-0@lo:400:kworker/u8:2.0 Request processed in 14us (711us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212708:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4877 00000100:00100000:0.0:1761315861.212714:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4878 00000100:00100000:0.0:1761315861.212717:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b79651d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322600192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212738:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b79651d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322600192:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212742:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4878 00000100:00100000:0.0:1761315861.212756:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600640 00000100:00100000:0.0:1761315861.212771:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600576 00000100:00100000:0.0:1761315861.212780:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600320 00000100:00100000:0.0:1761315861.212786:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600384 00000100:00100000:0.0:1761315861.212792:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1585 00000100:00100000:0.0:1761315861.212795:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322600640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212814:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322600640:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (380us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212819:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1585 00000100:00100000:0.0:1761315861.212826:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1586 00000100:00100000:0.0:1761315861.212829:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322600576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.212849:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322600576:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315861.212854:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1586 00000100:00100000:0.0:1761315861.212863:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2399 00000100:00100000:0.0:1761315861.212866:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322600320:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315861.212950:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315861.212980:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322600320:12345-0@lo:8:kworker/u8:2.0 Request processed in 113us (457us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315861.212986:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2399 00000100:00100000:0.0:1761315861.212996:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2400 00000100:00100000:0.0:1761315861.212998:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322600384:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315861.213676:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600256 00000100:00100000:3.0:1761315861.213706:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3046 00000100:00100000:3.0:1761315861.213709:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322600256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315861.213747:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322600256:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (1340us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315861.213752:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3046 00000100:00100000:3.0:1761315861.213793:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322600704:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315861.213817:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315861.213837:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600704 00000100:00100000:3.0:1761315861.213850:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3047 00000100:00100000:3.0:1761315861.213853:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322600704:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315861.213872:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315861.213894:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322600704:12345-0@lo:41:osp-pre-0-1.0 Request processed in 40us (77us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315861.213902:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3047 00000100:00100000:3.0:1761315861.213929:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322600704:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315861.214007:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a07dadbc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322600192:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315861.214019:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c123309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322600576:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315861.214027:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322600256:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315861.217181:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315861.217273:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322600384:12345-0@lo:8:kworker/u8:2.0 Request processed in 4271us (4724us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315861.217290:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2400 00000100:00080000:0.0:1761315861.217375:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315861.217380:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315861.217385:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c14c604d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322600320:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315861.217395:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315861.217398:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315861.217400:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322600384:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315861.217467:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322600064:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.217478:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322600448:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315861.217488:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322600512:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315861.217583:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087e39f9c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322600640:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.217688:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600000 00000100:00100000:2.0:1761315861.217795:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1794 00000100:00100000:2.0:1761315861.217800:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322600000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315861.217856:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322600000:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (5514us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315861.217862:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1794 00000100:00100000:3.0:1761315861.218560:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315861.218573:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322600768:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315861.218611:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315861.218676:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600768 00000100:00100000:2.0:1761315861.218710:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1047 00000100:00100000:2.0:1761315861.218713:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322600768:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315861.218735:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315861.218768:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322600768:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (159us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315861.218774:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1047 00000100:00100000:2.0:1761315861.218802:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322600768:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315861.219520:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322600000:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761315862.238087:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315863.240018:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315863.240106:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315863.240139:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315863.240220:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570048 00000100:00100000:1.0:1761315863.240227:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569984 00000100:00100000:1.0:1761315863.240292:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4879 00000100:00100000:2.0:1761315863.240302:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4880 00000100:00100000:1.0:1761315863.240303:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307569984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315863.240311:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7904d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307570048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315863.240410:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7904d34 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307570048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 95us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315863.240411:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fac9fbed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307569984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 105us (265us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315863.240424:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4880 00000100:00100000:2.0:1761315863.240556:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307569920 00000100:00100000:2.0:1761315863.240580:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1795 00000100:00100000:2.0:1761315863.240585:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307569920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315863.240616:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307569920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 29us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315863.240622:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1795 00000100:00100000:3.0:1761315863.240652:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315863.240679:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570112 00000100:00100000:2.0:1761315863.240706:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2401 00000100:00100000:2.0:1761315863.240711:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a141c006 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307570112:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761315863.240780:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4879 00010000:02020000:2.0:1761315863.240801:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315863.240847:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a141c006 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307570112:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 135us (196us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315863.240854:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2401 00010000:00010000:0.0:1761315863.259567:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315863.771874:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3c5b3a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322600832:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315863.771947:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315863.773581:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600832 00000100:00100000:2.0:1761315863.773647:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3048 00000100:00100000:2.0:1761315863.773654:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322600832:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315863.773689:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315863.774235:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322600832:12345-0@lo:41:osp-pre-1-0.0 Request processed in 580us (2287us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315863.774243:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3048 00000100:00100000:1.0:1761315863.774306:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3c5b3a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322600832:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315864.283608:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315865.307531:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315866.331540:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315866.331599:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322600896:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315866.331665:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315866.331766:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600896 00000100:00100000:3.0:1761315866.331818:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3049 00000100:00100000:3.0:1761315866.331824:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322600896:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315866.331859:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315866.331912:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322600896:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315866.331919:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3049 00000100:00100000:1.0:1761315866.332242:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322600896:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315866.332604:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315866.333420:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322600960:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315866.333471:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315866.333603:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322601024:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315866.333629:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315866.333683:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601024 00000100:00100000:2.0:1761315866.333719:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1796 00000100:00100000:2.0:1761315866.333724:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322601024:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315866.333727:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322600960 00000100:00100000:1.0:1761315866.333755:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1048 00000100:00100000:1.0:1761315866.333758:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322600960:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315866.333910:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315866.333915:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315866.333918:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315866.333922:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4108 00000100:00080000:3.0:1761315866.333925:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315866.333928:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315866.333950:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bae14e16 x1846869322601280/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315866.333964:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315866.333967:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315866.333968:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315866.333971:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4108 00002000:00000024:1.0:1761315866.334455:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315866.334470:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322601024:12345-0@lo:400:kworker/u8:0.0 Request processed in 745us (839us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315866.334478:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1796 00000100:00100000:1.0:1761315866.334491:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322600960:12345-0@lo:13:osp-pre-1-1.0 Request processed in 732us (1020us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315866.334497:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1048 00000100:00100000:1.0:1761315866.334514:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322601152:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.334517:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322601216:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334540:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315866.334540:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315866.334557:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315866.334562:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322601088:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.334579:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bae14e16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322601280:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334582:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315866.334602:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315866.334605:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601152 00000100:00100000:2.0:1761315866.334624:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601280 00000100:00100000:1.0:1761315866.334629:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601088 00000100:00100000:2.0:1761315866.334641:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2402 00000100:00100000:1.0:1761315866.334644:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4881 00000100:00100000:2.0:1761315866.334644:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322601280:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334647:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322601152:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334676:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b52f64ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322601152:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (137us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315866.334681:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4881 00000100:00100000:1.0:1761315866.334691:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4882 00000100:00100000:1.0:1761315866.334694:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322601088:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315866.334728:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315866.334743:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e2af60 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322601088:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (161us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315866.334749:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4882 00000100:00100000:2.0:1761315866.334753:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322601280:12345-0@lo:8:kworker/u8:0.0 Request processed in 108us (152us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315866.334758:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2402 00000100:00100000:1.0:1761315866.334780:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322601088:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315866.334780:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315866.334783:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315866.334786:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bae14e16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322601280:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334788:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322601152:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315866.334811:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601216 00000100:00080000:3.0:1761315866.334858:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315866.334862:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315866.334895:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aff84b27 x1846869322601344/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315866.334920:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322601344:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315866.334942:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315866.334950:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efeef3de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322601472:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.334962:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601344 00000100:00100000:3.0:1761315866.334973:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315866.334977:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2403 00000100:00100000:2.0:1761315866.334980:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322601344:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315866.334989:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315866.334992:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005167c36d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322601536:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315866.335023:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315866.335051:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601536 00000100:00100000:3.0:1761315866.335070:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1587 00010000:02020000:2.0:1761315866.335072:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315866.335074:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322601536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335097:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322601344:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (155us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315866.335097:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322601536:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (74us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315866.335103:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1587 00000100:00100000:0.0:1761315866.335147:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322601024:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315866.335163:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322600960:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315866.335180:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006729a9ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322601408:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315866.335200:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315866.335215:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2403 00000100:00100000:2.0:1761315866.335232:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601472 00000100:00100000:0.0:1761315866.335236:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3050 00000100:00100000:2.0:1761315866.335251:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601408 00000100:00100000:2.0:1761315866.335265:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4883 00000100:00100000:2.0:1761315866.335268:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efdaddff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322601472:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335296:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efdaddff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322601472:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315866.335301:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4883 00000100:00100000:2.0:1761315866.335312:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4884 00000100:00100000:2.0:1761315866.335315:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322601408:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315866.335326:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322601216:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335359:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea7e262a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322601408:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (142us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315866.335365:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4884 00000100:00100000:0.0:1761315866.335369:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322601216:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (831us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315866.335374:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3050 00000100:00080000:2.0:1761315866.335390:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315866.335393:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315866.335396:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322601344:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335414:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006729a9ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322601408:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315866.335416:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322601216:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335436:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005167c36d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322601536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315866.335444:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efeef3de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322601472:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315867.356045:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315868.366339:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315868.366413:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315868.366445:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315868.366545:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315868.366583:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570368 00000100:00100000:1.0:1761315868.366601:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570304 00000100:00100000:2.0:1761315868.366622:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2404 00000100:00100000:2.0:1761315868.366629:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307570368:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:1.0:1761315868.366632:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570240 00000100:00100000:1.0:1761315868.366648:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4885 00000100:00100000:1.0:1761315868.366652:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c123309 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315868.366716:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c123309 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315868.366723:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4885 00000100:00100000:1.0:1761315868.366739:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4886 00000100:00100000:1.0:1761315868.366743:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a07dadbc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570240:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315868.366765:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a07dadbc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570240:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315868.366770:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4886 00010000:02020000:2.0:1761315868.366918:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315868.367039:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570176 00000100:00100000:3.0:1761315868.367073:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1797 00000100:00100000:3.0:1761315868.367081:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570176:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315868.367179:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307570368:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 550us (634us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315868.367187:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2404 00000100:00100000:3.0:1761315868.368074:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570176:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 993us (1741us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315868.368084:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1797 00010000:00010000:2.0:1761315868.379880:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315868.891642:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322601600:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315868.891710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315868.891787:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601600 00000100:00100000:1.0:1761315868.893366:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3051 00000100:00100000:1.0:1761315868.893377:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322601600:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315868.893419:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315868.894424:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322601600:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1044us (2711us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315868.894435:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3051 00000100:00100000:0.0:1761315868.894735:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322601600:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315869.403750:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315870.427825:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315871.451687:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315871.451694:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315871.451697:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315871.451705:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4113 00000100:00080000:3.0:1761315871.451708:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315871.451714:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315871.451732:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006729a9ea x1846869322601984/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315871.451746:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315871.451748:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315871.451749:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315871.451752:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4113 00000100:00080000:3.0:1761315871.451754:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315871.451756:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315871.451768:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aff84b27 x1846869322602048/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315871.451811:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322601664:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315871.451875:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315871.451893:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322601920:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315871.451914:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315871.451925:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bae14e16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322602304:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315871.451945:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315871.451962:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315871.451999:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005167c36d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322601856:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315871.452022:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315871.452034:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315871.452053:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315871.452135:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322601728:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452164:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315871.452188:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322602112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452207:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315871.452222:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315871.452225:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efeef3de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322601792:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452245:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315871.452256:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322602176:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452275:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315871.452309:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602304 00000100:00100000:0.0:1761315871.452371:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602240 00000100:00100000:0.0:1761315871.452392:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1588 00000100:00100000:0.0:1761315871.452397:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322602304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452457:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322602304:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (511us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452464:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1588 00000100:00100000:0.0:1761315871.452481:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1589 00000100:00100000:0.0:1761315871.452485:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322602240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452507:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322602240:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (454us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452512:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1589 00000100:00100000:0.0:1761315871.452532:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601856 00000100:00100000:0.0:1761315871.452555:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601728 00000100:00100000:0.0:1761315871.452570:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602112 00000100:00100000:0.0:1761315871.452578:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602176 00000100:00100000:0.0:1761315871.452590:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4887 00000100:00100000:0.0:1761315871.452593:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322601856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452628:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322601856:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (606us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452633:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4887 00000100:00100000:0.0:1761315871.452644:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4888 00000100:00100000:0.0:1761315871.452647:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fd6d9470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322601728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452672:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fd6d9470 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322601728:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (509us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452678:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4888 00000100:00100000:0.0:1761315871.452688:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4889 00000100:00100000:0.0:1761315871.452692:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c28f1f32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322602112:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452715:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c28f1f32 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322602112:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (507us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452720:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4889 00000100:00100000:0.0:1761315871.452729:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4890 00000100:00100000:0.0:1761315871.452732:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e0e11476 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322602176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452755:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e0e11476 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322602176:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (480us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315871.452761:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4890 00000100:00100000:0.0:1761315871.452805:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322602176:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452817:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322601728:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452825:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322602112:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452855:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005167c36d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322601856:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452864:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315871.452871:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bae14e16 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322602304:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761315871.454196:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315871.454219:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006729a9ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322601984:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315871.454278:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315871.454317:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322602048:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315871.454340:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315871.454425:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601664 00000100:00100000:2.0:1761315871.454465:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1798 00000100:00100000:2.0:1761315871.454470:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322601664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315871.454518:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322601664:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (2644us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315871.454525:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1798 00000100:00100000:2.0:1761315871.454555:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601984 00000100:00100000:2.0:1761315871.454570:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602048 00000100:00100000:2.0:1761315871.454582:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2405 00000100:00100000:2.0:1761315871.454585:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322601984:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315871.454669:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315871.454699:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322601984:12345-0@lo:8:kworker/u8:0.0 Request processed in 113us (422us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315871.454705:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2405 00000100:00100000:2.0:1761315871.454714:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2406 00000100:00100000:2.0:1761315871.454718:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322602048:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315871.454789:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315871.454809:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322602048:12345-0@lo:8:kworker/u8:0.0 Request processed in 90us (469us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315871.454814:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2406 00000100:00100000:2.0:1761315871.454833:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601920 00000100:00100000:2.0:1761315871.454851:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322601792 00000100:00100000:2.0:1761315871.454866:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3052 00000100:00100000:2.0:1761315871.454870:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322601920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315871.454897:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322601920:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (2982us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315871.454902:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3052 00000100:00100000:2.0:1761315871.454912:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3053 00000100:00100000:2.0:1761315871.454916:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322601792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315871.455007:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322602368:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315871.455058:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322602432:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315871.459943:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315871.460563:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efeef3de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322601792:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315871.460625:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602368 00000100:00100000:1.0:1761315871.460666:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1049 00000100:00100000:1.0:1761315871.460671:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322602368:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315871.460689:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322601664:0@lo:400:kworker/u8:0.0 00002000:00000024:1.0:1761315871.460702:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315871.460702:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322601920:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315871.460740:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322602368:12345-0@lo:13:osp-pre-1-1.0 Request processed in 68us (804us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315871.460746:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1049 00000100:00080000:3.0:1761315871.460749:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315871.460752:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315871.460756:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006729a9ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322601984:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315871.460765:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315871.460767:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315871.460770:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322602048:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315871.460779:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322602368:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315871.460918:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315871.461150:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602432 00000100:00100000:3.0:1761315871.461184:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3054 00000100:00100000:3.0:1761315871.461188:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322602432:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315871.461214:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315871.462153:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322601792:12345-0@lo:400:kworker/u8:0.0 Request processed in 7234us (9905us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315871.462162:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3053 00000100:00100000:1.0:1761315871.463666:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322602432:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315871.467484:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322602432:12345-0@lo:41:osp-pre-0-1.0 Request processed in 6285us (6557us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315871.467500:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3054 00010000:00010000:0.0:1761315872.475572:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315873.485226:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315873.485276:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315873.485312:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315873.485377:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315873.485458:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570624 00000100:00100000:0.0:1761315873.485506:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2407 00000100:00100000:0.0:1761315873.485513:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307570624:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315873.485620:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315873.485678:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307570624:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 164us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315873.485685:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2407 00000100:00100000:0.0:1761315873.485705:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570432 00000100:00100000:0.0:1761315873.485728:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1799 00000100:00100000:0.0:1761315873.485732:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570432:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315873.485763:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570432:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 30us (488us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315873.485769:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1799 00000100:00100000:1.0:1761315873.486017:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570496 00000100:00100000:1.0:1761315873.486049:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570560 00000100:00100000:1.0:1761315873.486066:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4891 00000100:00100000:1.0:1761315873.486070:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c14c604d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570496:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315873.486100:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c14c604d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570496:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 30us (884us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315873.486105:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4891 00000100:00100000:1.0:1761315873.486115:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4892 00000100:00100000:1.0:1761315873.486118:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fba44d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315873.486133:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fba44d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 14us (823us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315873.486138:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4892 00010000:00010000:0.0:1761315873.499485:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315874.011968:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315874.011996:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322602496:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315874.012074:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315874.012336:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602496 00000100:00100000:2.0:1761315874.013239:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3055 00000100:00100000:2.0:1761315874.013247:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322602496:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315874.013287:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315874.013378:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322602496:12345-0@lo:41:osp-pre-1-0.0 Request processed in 130us (1304us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315874.013385:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3055 00000100:00100000:1.0:1761315874.013430:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322602496:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315874.523671:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315875.548387:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315876.571633:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602560:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315876.571662:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315876.571714:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315876.571950:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315876.571960:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322602624:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315876.571990:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315876.572375:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315876.572380:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315876.572383:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315876.572387:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4119 00000100:00080000:3.0:1761315876.572390:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315876.572396:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315876.572412:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bf1342a6 x1846869322603008/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315876.572425:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315876.572428:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315876.572429:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315876.572431:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4119 00000100:00080000:3.0:1761315876.572433:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315876.572435:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315876.572442:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000885db3f x1846869322603072/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315876.572472:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322603008:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315876.572498:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.572513:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322603072:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315876.572526:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.572548:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322602880:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.572567:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.572578:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322603264:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.572595:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.572610:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315876.572615:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008e76633c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322602688:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315876.572621:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602560 00000100:00100000:3.0:1761315876.572637:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.572650:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322602944:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315876.572658:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1050 00000100:00100000:0.0:1761315876.572664:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322602560:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315876.572674:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602880 00002000:00000024:0.0:1761315876.572692:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315876.572698:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4893 00000100:00100000:3.0:1761315876.572701:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067bcaad5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322602880:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315876.572724:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322602560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 61us (1011us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315876.572730:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067bcaad5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322602880:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (163us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315876.572731:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1050 00000100:00100000:3.0:1761315876.572735:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4893 00000100:00100000:3.0:1761315876.572752:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603008 00000100:00100000:0.0:1761315876.572754:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603072 00000100:00100000:3.0:1761315876.572765:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603264 00000100:00100000:1.0:1761315876.573105:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602624 00000100:00100000:1.0:1761315876.573137:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3056 00000100:00100000:1.0:1761315876.573140:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322602624:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315876.573161:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315876.573196:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322602624:12345-0@lo:41:osp-pre-0-1.0 Request processed in 54us (1205us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315876.573202:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3056 00000100:00100000:1.0:1761315876.573226:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573249:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315876.573263:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091728e30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322603200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573284:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315876.573318:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ebcc50d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602560:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315876.573336:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315876.573341:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322602752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573377:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315876.573391:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322603136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573409:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315876.573431:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603200 00000100:00100000:1.0:1761315876.573452:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602752 00000100:00100000:1.0:1761315876.573466:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603136 00000100:00100000:1.0:1761315876.573480:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4894 00000100:00100000:1.0:1761315876.573484:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322603200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573515:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f990ca7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322603200:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315876.573521:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4894 00000100:00100000:1.0:1761315876.573531:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4895 00000100:00100000:1.0:1761315876.573534:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322602752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573557:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c3594b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322602752:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315876.573562:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4895 00000100:00100000:1.0:1761315876.573572:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1761315876.573575:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322603136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573598:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d5431e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322603136:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (188us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315876.573603:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4896 00000100:00100000:1.0:1761315876.573628:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322602752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573637:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322603136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573645:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091728e30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322603200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573667:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602816 00000100:00100000:1.0:1761315876.573686:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3057 00000100:00100000:1.0:1761315876.573690:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322602816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315876.573718:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322602816:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (469us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315876.573727:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3057 00000100:00100000:1.0:1761315876.573747:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322602816:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315876.573893:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1590 00000100:00100000:0.0:1761315876.573898:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322603264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315876.573942:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322603264:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (1347us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315876.573948:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1590 00000100:00100000:0.0:1761315876.573960:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2408 00000100:00100000:0.0:1761315876.573963:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322603008:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574298:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2409 00000100:00100000:3.0:1761315876.574303:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322603072:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315876.574390:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315876.574424:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322603072:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (1897us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315876.574430:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2409 00000100:00080000:3.0:1761315876.574456:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315876.574459:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315876.574462:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322603072:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574500:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322602624:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315876.574533:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602688 00000100:00100000:3.0:1761315876.574554:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1800 00000100:00100000:3.0:1761315876.574558:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322602688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574584:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322602688:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1947us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315876.574589:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1800 00000100:00100000:3.0:1761315876.574614:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315876.574638:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322602880:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574646:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322603264:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574663:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008e76633c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322602688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574684:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322602944 00000100:00100000:3.0:1761315876.574703:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3058 00000100:00100000:3.0:1761315876.574706:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322602944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315876.574732:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322602944:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (118us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315876.574737:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3058 00000100:00100000:3.0:1761315876.574755:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322602944:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761315876.575399:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315876.575438:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322603008:12345-0@lo:8:kworker/u8:2.0 Request processed in 1473us (2939us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315876.575445:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2408 00000100:00080000:3.0:1761315876.579047:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315876.579057:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315876.579064:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322603008:0@lo:8:kworker/u8:2.0 00010000:00010000:1.0:1761315877.595794:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315878.601460:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315878.601521:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315878.601540:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570880 00000100:00100000:2.0:1761315878.601560:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315878.601584:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2410 00000100:00100000:1.0:1761315878.601591:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307570880:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:2.0:1761315878.601604:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315878.601627:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570816 00000100:00100000:3.0:1761315878.601665:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570752 00000100:00100000:2.0:1761315878.601673:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4897 00010000:02020000:1.0:1761315878.601685:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315878.601687:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315878.601700:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4898 00000100:00100000:3.0:1761315878.601705:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315878.601744:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307570880:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 153us (291us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315878.601753:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2410 00000100:00100000:1.0:1761315878.601794:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570688 00000100:00100000:1.0:1761315878.601817:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1801 00000100:00100000:1.0:1761315878.601821:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315878.602084:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004efff1f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307570816:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 402us (524us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315878.602088:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ebcc50d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307570752:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 379us (483us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315878.602097:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4897 00000100:00100000:3.0:1761315878.602104:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4898 00000100:00100000:1.0:1761315878.605947:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 4115us (4418us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315878.605964:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1801 00010000:00010000:0.0:1761315878.619491:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315879.131869:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315879.133094:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322603328:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315879.133175:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315879.133270:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603328 00000100:00100000:3.0:1761315879.133321:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3059 00000100:00100000:3.0:1761315879.133327:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322603328:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315879.133371:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315879.133419:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322603328:12345-0@lo:41:osp-pre-1-0.0 Request processed in 92us (245us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315879.133427:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3059 00000100:00100000:0.0:1761315879.133452:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322603328:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315879.643521:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315880.668725:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315881.691582:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322603392:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315881.691636:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315881.691652:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315881.691763:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603392 00000100:00100000:0.0:1761315881.691810:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1051 00000100:00100000:0.0:1761315881.691816:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322603392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315881.691843:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315881.691879:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322603392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (230us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315881.691885:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1051 00000100:00100000:1.0:1761315881.691979:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072a20377 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322603392:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315881.692590:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322603456:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315881.692643:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315881.692753:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603456 00000100:00100000:2.0:1761315881.692755:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315881.692789:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3060 00000100:00100000:1.0:1761315881.692796:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee3f3653 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322603456:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315881.692823:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315881.692876:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee3f3653 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322603456:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315881.692882:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3060 00000100:00100000:2.0:1761315881.693295:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322603520:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315881.693331:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315881.693420:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322603456:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315881.693685:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603520 00000100:00100000:2.0:1761315881.694251:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1802 00000100:00100000:2.0:1761315881.694257:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322603520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315881.694320:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322603520:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (989us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315881.694326:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1802 00000100:00100000:1.0:1761315881.694580:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322603520:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315881.694627:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322603584:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315881.694663:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315881.694718:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315881.694724:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315881.694728:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315881.694738:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4124 00000100:00080000:0.0:1761315881.694741:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315881.694745:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315881.694765:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006fc130f2 x1846869322603776/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315881.694780:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315881.694784:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315881.694785:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315881.694788:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4124 00000100:00080000:0.0:1761315881.694790:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315881.694792:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315881.694803:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e62725a5 x1846869322603840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315881.694857:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322603712:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.694929:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315881.694951:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761315881.694956:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322603648:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.694991:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603712 00000100:00100000:0.0:1761315881.695020:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3061 00000100:00100000:0.0:1761315881.695025:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322603712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.695059:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322603712:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (130us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315881.695065:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3061 00000100:00100000:0.0:1761315881.695086:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322603904:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.695133:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322603776:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315881.695160:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315881.695174:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322603840:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315881.695196:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315881.695234:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603776 00000100:00100000:0.0:1761315881.695252:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603840 00000100:00100000:0.0:1761315881.695267:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2411 00000100:00100000:0.0:1761315881.695271:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006acadeb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322603776:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315881.695357:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2412 00000100:00100000:3.0:1761315881.695362:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322603840:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315881.695451:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315881.695486:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322603840:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (289us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315881.695492:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2412 00000100:00080000:3.0:1761315881.695524:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315881.695528:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315881.695532:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322603840:0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761315881.695970:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315881.696001:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006acadeb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322603776:12345-0@lo:8:kworker/u8:0.0 Request processed in 729us (841us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315881.696008:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2411 00000100:00080000:3.0:1761315881.696102:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315881.696106:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315881.696109:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322603776:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315881.697942:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603584 00000100:00100000:2.0:1761315881.698022:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4899 00000100:00100000:2.0:1761315881.698029:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322603584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315881.698105:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322603584:12345-0@lo:400:kworker/u8:0.0 Request processed in 76us (3443us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315881.698113:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4899 00000100:00100000:0.0:1761315881.700088:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315881.700145:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315881.700157:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322603968:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.700209:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315881.700319:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603904 00000100:00100000:3.0:1761315881.700384:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603968 00000100:00100000:3.0:1761315881.700407:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4900 00000100:00100000:3.0:1761315881.700412:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a141c006 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322603904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315881.700466:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a141c006 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322603904:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (385us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315881.700473:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4900 00000100:00100000:3.0:1761315881.700493:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4901 00000100:00100000:3.0:1761315881.700497:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322603968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315881.700523:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006536d946 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322603968:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315881.700528:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4901 00000100:00100000:3.0:1761315881.700562:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322603968:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315881.700574:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322603904:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315881.700758:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322603584:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315881.700802:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315881.700824:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604032:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315881.700849:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315881.700881:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322603712:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315881.700944:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604032 00000100:00100000:1.0:1761315881.700975:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1591 00000100:00100000:1.0:1761315881.700979:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604032:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315881.701023:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604032:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (173us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315881.701029:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1591 00000100:00100000:2.0:1761315881.701092:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604032:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.701156:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322603648 00000100:00100000:0.0:1761315881.701190:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4902 00000100:00100000:0.0:1761315881.701193:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322603648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315881.701235:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000772f557b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322603648:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (434us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315881.701241:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4902 00000100:00100000:2.0:1761315881.701302:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322603648:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315882.715523:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315883.721049:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315883.721095:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315883.721193:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571136 00000100:00100000:3.0:1761315883.721242:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2413 00000100:00100000:3.0:1761315883.721249:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571136:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761315883.721372:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315883.721444:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571136:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 194us (403us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315883.721452:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2413 00000100:00100000:0.0:1761315883.721583:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571008 00000100:00100000:0.0:1761315883.721625:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4903 00000100:00100000:0.0:1761315883.721629:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307571008:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315883.721684:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000572cc7ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307571008:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 54us (590us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315883.721691:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4903 00000100:00100000:2.0:1761315883.722092:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315883.722145:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315883.722194:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571072 00000100:00100000:0.0:1761315883.722221:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307570944 00000100:00100000:1.0:1761315883.722242:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4904 00000100:00100000:1.0:1761315883.722251:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307571072:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315883.722258:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1803 00000100:00100000:0.0:1761315883.722263:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570944:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315883.722299:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072a20377 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307571072:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 47us (152us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315883.722301:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307570944:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (207us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315883.722317:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1803 00000100:00100000:1.0:1761315883.722317:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4904 00010000:00010000:1.0:1761315883.740013:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315884.252052:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000482873b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604096:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315884.252125:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315884.252981:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604096 00000100:00100000:1.0:1761315884.253042:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3062 00000100:00100000:1.0:1761315884.253049:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604096:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315884.253082:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315884.253156:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604096:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (1032us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315884.253164:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3062 00000100:00100000:2.0:1761315884.253263:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000482873b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604096:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315884.765564:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315885.788028:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315886.811604:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315886.811604:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322604352:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315886.811612:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315886.811617:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315886.811622:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4129 00000100:00080000:1.0:1761315886.811625:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315886.811631:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315886.811653:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a10b3b20 x1846869322604608/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315886.811667:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315886.811671:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315886.811672:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315886.811676:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4129 00000100:00080000:1.0:1761315886.811679:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315886.811680:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315886.811685:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315886.811696:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000077564dac x1846869322604672/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315886.811717:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322604736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.811745:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315886.811760:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:2.0:1761315886.811765:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322604288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.811792:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315886.811803:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315886.811817:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604544:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315886.811828:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604352 00000100:00100000:2.0:1761315886.811848:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315886.811880:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604928:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.811906:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315886.811909:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604736 00000100:00100000:2.0:1761315886.811942:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4905 00000100:00100000:0.0:1761315886.811952:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4906 00000100:00100000:2.0:1761315886.811952:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322604352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315886.811963:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322604736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315886.812000:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604544 00000100:00100000:2.0:1761315886.812009:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322604352:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (332us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315886.812017:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322604736:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (270us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315886.812020:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4905 00000100:00100000:0.0:1761315886.812030:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4906 00000100:00100000:1.0:1761315886.812036:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3063 00000100:00100000:1.0:1761315886.812042:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812054:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604288 00000100:00100000:3.0:1761315886.812055:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604160:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315886.812091:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1804 00000100:00100000:1.0:1761315886.812096:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604544:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (247us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315886.812098:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322604288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.812105:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315886.812112:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3063 00000100:00100000:1.0:1761315886.812170:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604928 00000100:00100000:3.0:1761315886.812181:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322604480:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812188:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322604288:12345-0@lo:400:kworker/u8:0.0 Request processed in 89us (396us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315886.812196:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1804 00000100:00100000:1.0:1761315886.812208:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1592 00000100:00100000:1.0:1761315886.812217:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604928:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.812218:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.812247:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b10b1dae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604864:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812249:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000885db3f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322604288:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315886.812257:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604480 00000100:00100000:2.0:1761315886.812270:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006026bd47 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604544:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812283:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5918643 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322604928:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315886.812290:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4907 00000100:00100000:2.0:1761315886.812296:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322604736:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315886.812297:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000482873b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322604480:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812321:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7bedaeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322604352:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315886.812391:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604160 00000100:00100000:2.0:1761315886.812416:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1052 00000100:00100000:2.0:1761315886.812420:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322604160:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315886.812440:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315886.812960:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.812981:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315886.812986:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604224:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315886.813009:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.813023:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604416:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813039:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.813052:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322604800:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813069:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.813093:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604864 00000100:00100000:3.0:1761315886.813114:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1593 00000100:00100000:3.0:1761315886.813118:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322604864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813150:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322604864:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (191us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315886.813156:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1593 00000100:00100000:3.0:1761315886.813176:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604800 00000100:00100000:3.0:1761315886.813198:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4908 00000100:00100000:3.0:1761315886.813201:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322604800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813228:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a460ec1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322604800:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315886.813233:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4908 00000100:00100000:3.0:1761315886.813264:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322604800:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813274:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b10b1dae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604864:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813296:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604224 00000100:00100000:3.0:1761315886.813315:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604416 00000100:00100000:3.0:1761315886.813332:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3064 00000100:00100000:3.0:1761315886.813336:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322604224:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315886.813373:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315886.813404:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322604224:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315886.813414:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3064 00000100:00100000:3.0:1761315886.813427:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3065 00000100:00100000:3.0:1761315886.813431:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322604416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813454:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322604416:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (414us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315886.813460:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3065 00000100:00100000:3.0:1761315886.813490:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093a69a62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604224:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315886.813499:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fe1c4fae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604416:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813544:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322604608:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813570:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315886.813589:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322604672:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315886.813604:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000482873b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322604480:12345-0@lo:400:kworker/u8:0.0 Request processed in 1306us (1389us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315886.813614:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4907 00000100:00100000:3.0:1761315886.813620:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315886.813631:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322604160:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1207us (1529us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315886.813640:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1052 00000100:00100000:0.0:1761315886.813661:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322604160:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315886.813661:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604608 00000100:00100000:0.0:1761315886.813679:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322604480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813687:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604672 00000100:00100000:2.0:1761315886.813701:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2414 00000100:00100000:2.0:1761315886.813707:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322604608:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315886.813713:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2415 00000100:00100000:3.0:1761315886.813718:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322604672:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315886.814323:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:2.0:1761315886.814378:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315886.814409:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322604672:12345-0@lo:8:kworker/u8:0.0 Request processed in 688us (790us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315886.814415:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322604608:12345-0@lo:8:kworker/u8:0.0 Request processed in 708us (845us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315886.814420:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2415 00000100:00100000:2.0:1761315886.814427:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2414 00000100:00080000:0.0:1761315886.814780:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315886.814785:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315886.814790:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322604608:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315886.814801:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315886.814803:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315886.814806:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077564dac pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322604672:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315886.820387:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322604928:12345-0@lo:400:kworker/u8:0.0 Request processed in 8163us (8473us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315886.820404:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1592 00010000:00010000:0.0:1761315887.835545:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315888.842524:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315888.842574:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315888.842605:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571392 00000100:00100000:2.0:1761315888.842609:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315888.842659:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571200 00000100:00100000:2.0:1761315888.842684:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315888.842708:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1805 00000100:00100000:0.0:1761315888.842716:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307571200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315888.842778:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571264 00000100:00100000:0.0:1761315888.842798:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307571200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 82us (189us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315888.842807:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1805 00000100:00100000:3.0:1761315888.842812:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571328 00000100:00100000:3.0:1761315888.842834:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4909 00000100:00100000:3.0:1761315888.842840:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c227bf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307571264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315888.842876:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4910 00000100:00100000:0.0:1761315888.842883:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002672a249 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307571328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315888.842912:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c227bf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307571264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 71us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315888.842922:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002672a249 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307571328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (237us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315888.842924:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4909 00000100:00100000:0.0:1761315888.842932:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4910 00000100:00100000:1.0:1761315888.843449:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2416 00000100:00100000:1.0:1761315888.843454:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571392:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315888.843547:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315888.843590:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571392:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 134us (1072us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315888.843597:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2416 00010000:00010000:2.0:1761315888.859736:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315889.372135:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315889.373897:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322604992:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315889.374444:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315889.374946:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322604992 00000100:00100000:2.0:1761315889.375003:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3066 00000100:00100000:2.0:1761315889.375009:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322604992:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315889.375043:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315889.375097:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322604992:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (654us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315889.375104:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3066 00000100:00100000:1.0:1761315889.375804:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322604992:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315889.884848:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315890.907502:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315891.931512:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322605120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.931518:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605056:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761315891.931620:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315891.931625:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315891.931627:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315891.931632:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4134 00000100:00080000:3.0:1761315891.931634:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315891.931639:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315891.931655:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007c6c8f7d x1846869322605440/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315891.931669:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315891.931672:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315891.931673:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315891.931675:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4134 00000100:00080000:3.0:1761315891.931677:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315891.931680:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315891.931692:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d981f77 x1846869322605568/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315891.931710:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322605248:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315891.931745:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c6c8f7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322605440:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315891.932894:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.932935:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322605376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.932964:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.932982:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605760:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933008:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933023:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315891.933028:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322605312:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933050:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933065:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000015a4137c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322605696:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933084:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933109:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322605184:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933132:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933147:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322605504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315891.933168:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933184:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322605824:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933204:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933220:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315891.933225:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322605632:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933246:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.933278:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605760 00000100:00100000:1.0:1761315891.933331:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605824 00000100:00100000:1.0:1761315891.933375:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1594 00000100:00100000:1.0:1761315891.933380:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322605760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933437:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322605760:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933443:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1594 00000100:00100000:1.0:1761315891.933456:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1595 00000100:00100000:1.0:1761315891.933460:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322605824:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933487:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322605824:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933492:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1595 00000100:00100000:1.0:1761315891.933522:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322605824:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933549:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605760:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933580:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605504 00000100:00100000:1.0:1761315891.933601:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1053 00000100:00100000:1.0:1761315891.933604:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322605504:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315891.933624:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315891.933662:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322605504:12345-0@lo:13:osp-pre-1-1.0 Request processed in 56us (494us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933668:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1053 00000100:00100000:1.0:1761315891.933686:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605312 00000100:00100000:1.0:1761315891.933722:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605696 00000100:00100000:1.0:1761315891.933736:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605184 00000100:00100000:1.0:1761315891.933745:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605632 00000100:00100000:1.0:1761315891.933755:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4911 00000100:00100000:1.0:1761315891.933758:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322605312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933792:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5918643 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322605312:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (740us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933797:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4911 00000100:00100000:1.0:1761315891.933808:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4912 00000100:00100000:1.0:1761315891.933811:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322605696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933847:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322605696:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (763us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933853:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4912 00000100:00100000:1.0:1761315891.933862:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4913 00000100:00100000:1.0:1761315891.933867:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322605184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933890:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322605184:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (758us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933895:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4913 00000100:00100000:1.0:1761315891.933903:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4914 00000100:00100000:1.0:1761315891.933906:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322605632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933928:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322605632:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (681us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.933933:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4914 00000100:00100000:1.0:1761315891.933962:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322605632:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933972:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322605184:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.933984:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322605504:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315891.934012:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322605312:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.934020:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000015a4137c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322605696:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.934074:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605056 00000100:00100000:1.0:1761315891.934095:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605376 00000100:00100000:1.0:1761315891.934110:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3067 00000100:00100000:1.0:1761315891.934114:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322605056:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315891.934135:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315891.934166:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322605056:12345-0@lo:41:osp-pre-0-1.0 Request processed in 51us (1276us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.934173:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3067 00000100:00100000:1.0:1761315891.934183:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3068 00000100:00100000:1.0:1761315891.934186:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322605376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.934209:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322605376:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1246us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.934214:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3068 00000100:00100000:1.0:1761315891.934236:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322605376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.934248:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000422f974e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605056:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315891.935446:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761315891.935529:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315891.935869:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605120 00000100:00100000:1.0:1761315891.935923:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1806 00000100:00100000:1.0:1761315891.935929:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322605120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.936000:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322605120:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (557us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.936006:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1806 00000100:00100000:0.0:1761315891.936104:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322605120:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315891.936561:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315891.936567:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315891.936599:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d981f77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322605568:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315891.936626:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605440 00000100:00100000:3.0:1761315891.936630:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315891.937011:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605248 00000100:00100000:1.0:1761315891.937042:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3069 00000100:00100000:1.0:1761315891.937046:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322605248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315891.937088:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322605248:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (528us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315891.937094:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3069 00000100:00100000:3.0:1761315891.938279:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605568 00000100:00100000:3.0:1761315891.938315:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2417 00000100:00100000:3.0:1761315891.938319:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322605440:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315891.938417:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315891.938462:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322605440:12345-0@lo:8:kworker/u8:2.0 Request processed in 142us (1900us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315891.938469:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2417 00000100:00100000:3.0:1761315891.938481:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2418 00000100:00100000:3.0:1761315891.938484:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322605568:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315891.938564:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315891.938588:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322605568:12345-0@lo:8:kworker/u8:2.0 Request processed in 103us (1959us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315891.938593:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2418 00000100:00100000:3.0:1761315891.938641:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322605248:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761315891.940570:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315891.940578:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315891.940583:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c6c8f7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322605440:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761315891.940594:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315891.940596:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315891.940598:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d981f77 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322605568:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315892.955759:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315893.976150:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315893.976199:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315893.976232:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315893.976269:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315893.976340:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571520 00000100:00100000:3.0:1761315893.976430:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571584 00000100:00100000:3.0:1761315893.976450:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4915 00000100:00100000:3.0:1761315893.976457:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307571520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315893.976529:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307571520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 71us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315893.976536:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4915 00000100:00100000:3.0:1761315893.976552:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4916 00000100:00100000:3.0:1761315893.976556:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307571584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315893.976580:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077564dac pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307571584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (348us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315893.976590:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4916 00000100:00100000:0.0:1761315893.977677:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571456 00000100:00100000:0.0:1761315893.977719:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1807 00000100:00100000:0.0:1761315893.977724:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307571456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315893.977775:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307571456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 49us (1575us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315893.977781:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1807 00000100:00100000:1.0:1761315893.979147:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571648 00000100:00100000:1.0:1761315893.979193:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2419 00000100:00100000:1.0:1761315893.979203:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307571648:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315893.979294:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315893.979463:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307571648:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 263us (3193us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315893.979473:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2419 00010000:00010000:0.0:1761315893.980647:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315894.491691:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315894.491713:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322605888:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315894.491785:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315894.491904:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605888 00000100:00100000:3.0:1761315894.491949:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3070 00000100:00100000:3.0:1761315894.491955:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322605888:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315894.491984:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315894.492034:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322605888:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315894.492041:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3070 00000100:00100000:0.0:1761315894.492159:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004cf778b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322605888:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315895.003527:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315896.027910:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315897.051569:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315897.051576:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315897.051579:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315897.051587:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4139 00000100:00080000:1.0:1761315897.051590:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:3.0:1761315897.051593:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322606080:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315897.051596:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315897.051618:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060be72a7 x1846869322606400/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315897.051631:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315897.051634:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315897.051635:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315897.051638:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4139 00000100:00080000:1.0:1761315897.051639:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315897.051641:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315897.051650:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005e6dc415 x1846869322606464/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315897.051670:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315897.051691:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322606336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315897.051728:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315897.051758:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322606720:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315897.051782:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315897.051794:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606336 00000100:00100000:3.0:1761315897.051805:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 4 async RPCs [0->1] 00000100:00100000:3.0:1761315897.051809:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322606016:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315897.051830:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315897.051835:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3071 00000100:00100000:1.0:1761315897.051841:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322606336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315897.051841:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605952:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315897.051857:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315897.051868:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322606272:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.051886:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322606336:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315897.051887:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315897.051896:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3071 00000100:00100000:3.0:1761315897.051900:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322606656:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.051912:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322605952 00000100:00100000:3.0:1761315897.051915:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315897.051928:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3072 00000100:00100000:1.0:1761315897.051932:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322605952:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315897.051942:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322606336:0@lo:400:kworker/u8:0.0 00000004:00000024:1.0:1761315897.051955:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315897.051971:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606272 00000100:00100000:1.0:1761315897.051985:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322605952:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (128us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315897.051991:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3072 00000100:00100000:3.0:1761315897.051995:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4917 00000100:00100000:3.0:1761315897.051998:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c6c8f7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322606272:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.052012:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606720 00000100:00100000:3.0:1761315897.052024:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c6c8f7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322606272:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (136us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315897.052029:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4917 00000100:00100000:1.0:1761315897.052032:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606656 00000100:00100000:1.0:1761315897.052048:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1596 00000100:00100000:1.0:1761315897.052051:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322606720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315897.052051:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1597 00000100:00100000:3.0:1761315897.052054:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322606656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.052076:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322606720:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315897.052076:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322606656:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315897.052082:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1596 00000100:00100000:1.0:1761315897.052114:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322605952:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315897.052123:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322606272:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.052130:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322606656:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.052137:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322606720:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052245:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322606208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052270:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315897.052289:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322606592:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052306:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315897.052318:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315897.052322:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d6dfec7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322606144:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052336:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315897.052376:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322606528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052392:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315897.052412:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606080 00000100:00100000:2.0:1761315897.052431:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1808 00000100:00100000:2.0:1761315897.052434:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322606080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.052457:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322606080:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (788us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315897.052461:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1808 00000100:00100000:2.0:1761315897.052487:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606016 00000100:00100000:2.0:1761315897.052501:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1054 00000100:00100000:2.0:1761315897.052503:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322606016:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315897.052518:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315897.052537:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322606016:12345-0@lo:13:osp-pre-1-1.0 Request processed in 33us (708us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315897.052545:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1054 00000100:00100000:2.0:1761315897.052566:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606592 00000100:00100000:2.0:1761315897.052580:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606144 00000100:00100000:2.0:1761315897.052590:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606528 00000100:00100000:2.0:1761315897.052599:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4918 00000100:00100000:2.0:1761315897.052602:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322606592:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.052795:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4919 00000100:00100000:1.0:1761315897.052799:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322606144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315897.052878:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322606400:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315897.052904:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315897.052917:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322606464:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315897.052931:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315897.053060:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1597 00000100:00100000:3.0:1761315897.053076:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606400 00000100:00100000:3.0:1761315897.053088:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606464 00000100:00100000:3.0:1761315897.053099:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2420 00000100:00100000:3.0:1761315897.053149:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322606400:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315897.053243:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315897.053270:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322606400:12345-0@lo:8:kworker/u8:0.0 Request processed in 121us (366us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315897.053276:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2420 00000100:00100000:3.0:1761315897.053284:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2421 00000100:00100000:3.0:1761315897.053287:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322606464:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315897.053367:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315897.053383:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322606464:12345-0@lo:8:kworker/u8:0.0 Request processed in 96us (452us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315897.053391:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2421 00000100:00100000:3.0:1761315897.053405:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4920 00000100:00100000:3.0:1761315897.053408:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322606528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.053467:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322606144:12345-0@lo:400:kworker/u8:0.0 Request processed in 667us (1130us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315897.053467:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322606592:12345-0@lo:400:kworker/u8:0.0 Request processed in 864us (1160us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315897.053474:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4918 00000100:00100000:1.0:1761315897.053477:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4919 00000100:00100000:1.0:1761315897.053499:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606208 00000100:00100000:2.0:1761315897.053510:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d6dfec7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322606144:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.053519:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3073 00000100:00100000:1.0:1761315897.053522:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322606208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315897.053550:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322606208:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (1279us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315897.053555:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3073 00000100:00080000:0.0:1761315897.053556:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315897.053560:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315897.053563:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322606400:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315897.053571:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315897.053573:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315897.053575:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322606464:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315897.053586:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322606016:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315897.053595:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322606080:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315897.053962:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315897.054081:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322606528:12345-0@lo:400:kworker/u8:0.0 Request processed in 671us (1687us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315897.054087:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4920 00000100:00100000:2.0:1761315897.054155:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322606528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.054163:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322606208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315897.054171:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322606592:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315898.075523:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315899.081591:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315899.081689:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571904 00000100:00100000:2.0:1761315899.081734:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2422 00000100:00100000:2.0:1761315899.081740:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571904:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315899.081873:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315899.081943:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307571904:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 202us (359us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315899.081951:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2422 00000100:00100000:2.0:1761315899.082447:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315899.082478:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315899.082504:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315899.082524:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571776 00000100:00100000:2.0:1761315899.082548:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571840 00000100:00100000:2.0:1761315899.082562:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4921 00000100:00100000:2.0:1761315899.082566:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307571776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315899.082603:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307571776:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315899.082608:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4921 00000100:00100000:2.0:1761315899.082619:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4922 00000100:00100000:2.0:1761315899.082622:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307571840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315899.082637:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307571840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 14us (134us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315899.082642:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4922 00000100:00100000:1.0:1761315899.083062:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571712 00000100:00100000:1.0:1761315899.083094:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1809 00000100:00100000:1.0:1761315899.083098:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307571712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315899.083130:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307571712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315899.083136:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1809 00010000:00010000:0.0:1761315899.099520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315899.614556:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315899.614576:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322606784:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315899.614638:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315899.614817:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606784 00000100:00100000:3.0:1761315899.615182:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3074 00000100:00100000:3.0:1761315899.615188:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322606784:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315899.615220:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315899.615676:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322606784:12345-0@lo:41:osp-pre-1-0.0 Request processed in 487us (1038us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315899.615684:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3074 00000100:00100000:2.0:1761315899.615944:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322606784:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315900.124514:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315901.150327:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315902.171524:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315902.171531:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315902.171546:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322606848:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315902.171570:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322606912:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315902.171621:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315902.171622:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.171864:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606912 00000100:00100000:0.0:1761315902.171947:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606848 00000100:00100000:0.0:1761315902.171999:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3075 00000100:00100000:0.0:1761315902.172005:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322606848:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315902.172036:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315902.172088:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322606848:12345-0@lo:41:osp-pre-0-1.0 Request processed in 82us (474us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315902.172096:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3075 00000100:00100000:2.0:1761315902.172131:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322606848:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315902.172169:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1055 00000100:00100000:1.0:1761315902.172174:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322606912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315902.172195:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315902.172235:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322606912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 60us (621us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315902.172244:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1055 00000100:00100000:2.0:1761315902.172272:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322606912:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315902.172528:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322606976:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315902.172559:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315902.172630:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322606976 00000100:00100000:2.0:1761315902.172665:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1810 00000100:00100000:2.0:1761315902.172669:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322606976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177185:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322607168:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177257:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.177325:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607168 00000100:00100000:1.0:1761315902.177403:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3076 00000100:00100000:1.0:1761315902.177409:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322607168:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177461:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322607168:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (205us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315902.177469:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3076 00000100:00100000:1.0:1761315902.177509:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322607168:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177551:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322607104:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177596:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.177614:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315902.177620:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322607040:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177641:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.177666:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607104 00000100:00100000:1.0:1761315902.177689:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607040 00000100:00100000:1.0:1761315902.177708:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4923 00000100:00100000:1.0:1761315902.177711:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e29b36b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322607104:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177745:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e29b36b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322607104:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315902.177760:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4923 00000100:00100000:1.0:1761315902.177772:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4924 00000100:00100000:1.0:1761315902.177775:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc23aeea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322607040:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177801:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc23aeea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322607040:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315902.177806:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4924 00000100:00100000:1.0:1761315902.177838:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322607040:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.177846:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322607104:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315902.178971:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315902.178978:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315902.178981:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315902.178986:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4144 00000100:00080000:3.0:1761315902.178989:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315902.178993:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315902.179024:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f2415dea x1846869322607232/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315902.179041:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315902.179045:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315902.179046:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315902.179049:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4144 00000100:00080000:3.0:1761315902.179051:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315902.179054:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315902.179064:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000515354f3 x1846869322607296/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315902.179127:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315902.179138:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322607360:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315902.179179:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315902.179210:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607360 00000100:00100000:3.0:1761315902.179243:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4925 00000100:00100000:3.0:1761315902.179247:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322607360:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315902.179285:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322607360:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (107us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315902.179291:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4925 00000100:00100000:3.0:1761315902.179324:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322607360:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315902.179432:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322606976:12345-0@lo:400:kworker/u8:2.0 Request processed in 6760us (6871us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315902.179441:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1810 00000100:00100000:0.0:1761315902.182632:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322606976:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.185486:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322607232:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315902.185564:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.185600:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322607296:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315902.185622:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.185674:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322607488:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.185697:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315902.185730:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315902.185737:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322607424:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315902.185770:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315902.185892:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607232 00000100:00100000:3.0:1761315902.185928:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607296 00000100:00100000:3.0:1761315902.185940:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607488 00000100:00100000:3.0:1761315902.185961:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1598 00000100:00100000:3.0:1761315902.185966:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322607488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315902.186022:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322607488:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (325us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315902.186030:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1598 00000100:00100000:3.0:1761315902.186046:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2423 00000100:00100000:3.0:1761315902.186050:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322607232:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315902.186136:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315902.186170:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322607232:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (608us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315902.186177:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2423 00000100:00100000:3.0:1761315902.186187:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2424 00000100:00100000:3.0:1761315902.186190:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322607296:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315902.186267:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315902.186290:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322607296:12345-0@lo:8:kworker/u8:2.0 Request processed in 99us (668us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315902.186296:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2424 00000100:00100000:3.0:1761315902.186320:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607424 00000100:00100000:3.0:1761315902.186375:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4926 00000100:00100000:3.0:1761315902.186379:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322607424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315902.186409:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004eb8e25e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322607424:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (638us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315902.186414:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4926 00000100:00080000:3.0:1761315902.186448:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315902.186452:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315902.186456:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322607232:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315902.186465:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315902.186467:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315902.186469:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322607296:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315902.186499:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322607424:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315902.186507:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322607488:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315903.195610:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315904.201187:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315904.201274:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315904.201322:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315904.201441:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315904.202421:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572160 00000100:00100000:0.0:1761315904.202471:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2425 00000100:00100000:0.0:1761315904.202476:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307572160:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315904.202573:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315904.202642:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307572160:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 164us (1471us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315904.202649:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2425 00000100:00100000:3.0:1761315904.205755:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572096 00000100:00100000:3.0:1761315904.205825:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572032 00000100:00100000:3.0:1761315904.205859:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4927 00000100:00100000:3.0:1761315904.205866:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307572096:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315904.205942:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307572096:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 75us (4620us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315904.205949:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4927 00000100:00100000:3.0:1761315904.205974:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4928 00000100:00100000:3.0:1761315904.205978:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307572032:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315904.206004:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307572032:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 24us (4563us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315904.206010:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4928 00000100:00100000:2.0:1761315904.206012:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307571968 00000100:00100000:2.0:1761315904.206046:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1811 00000100:00100000:2.0:1761315904.206051:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307571968:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315904.206088:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307571968:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 36us (4814us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315904.206095:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1811 00010000:00010000:0.0:1761315904.219627:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315904.731593:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322607552:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315904.732010:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315904.732934:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607552 00000100:00100000:1.0:1761315904.732993:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3077 00000100:00100000:1.0:1761315904.733000:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efeef3de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322607552:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315904.733034:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315904.735522:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efeef3de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322607552:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2516us (3509us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315904.735536:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3077 00000100:00100000:3.0:1761315904.735861:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322607552:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315905.243557:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315906.267507:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315907.291507:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761315907.291512:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322607808:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315907.291514:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315907.291518:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315907.291524:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4149 00000100:00080000:3.0:1761315907.291526:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315907.291532:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315907.291552:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c18bc7a0 x1846869322608000/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315907.291566:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315907.291569:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315907.291570:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315907.291573:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4149 00000100:00080000:3.0:1761315907.291575:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761315907.291579:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315907.291580:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315907.291591:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000515354f3 x1846869322608064/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315907.291622:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322608192:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.291647:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322608000:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315907.291654:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315907.291674:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:3.0:1761315907.291685:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315907.291687:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322607616:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.291735:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315907.291741:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608000 00000100:00100000:3.0:1761315907.291742:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322608064:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315907.291751:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322607872:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.291776:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315907.291781:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315907.291802:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322608256:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.291817:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315907.291912:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322607680:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315907.291936:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315907.291952:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322607936:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.291967:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315907.291975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000005042c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322608320:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.291996:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607808 00000100:00100000:3.0:1761315907.292018:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322607744:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.292020:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607936 00000100:00100000:1.0:1761315907.292038:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3078 00000100:00100000:1.0:1761315907.292044:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322607808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292050:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315907.292066:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322608128:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.292092:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322607808:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (514us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315907.292097:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315907.292104:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3078 00000100:00100000:3.0:1761315907.292115:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322608384:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315907.292120:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3079 00000100:00100000:1.0:1761315907.292125:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322607936:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292138:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315907.292160:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322607936:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (192us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315907.292167:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3079 00000100:00100000:3.0:1761315907.292168:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608192 00000100:00100000:1.0:1761315907.292178:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608384 00000100:00100000:1.0:1761315907.292192:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3080 00000100:00100000:1.0:1761315907.292196:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322608384:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315907.292199:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607872 00000100:00100000:3.0:1761315907.292214:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607744 00000004:00000024:1.0:1761315907.292221:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315907.292224:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608128 00000100:00100000:3.0:1761315907.292238:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4929 00000100:00100000:3.0:1761315907.292243:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322608192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.292255:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322608384:12345-0@lo:41:osp-pre-0-1.0 Request processed in 57us (118us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315907.292262:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3080 00000100:00100000:3.0:1761315907.292280:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322608192:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (628us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315907.292285:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4930 00000100:00100000:1.0:1761315907.292292:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322607872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292292:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4929 00000100:00100000:3.0:1761315907.292310:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4931 00000100:00100000:3.0:1761315907.292315:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322607744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.292324:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322607872:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (544us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315907.292330:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4930 00000100:00100000:3.0:1761315907.292364:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f159976f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322607744:12345-0@lo:400:kworker/u8:2.0 Request processed in 47us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315907.292370:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4931 00000100:00100000:1.0:1761315907.292384:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4932 00000100:00100000:1.0:1761315907.292389:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322608128:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315907.292417:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d5bc7ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322608128:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315907.292422:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4932 00000100:00100000:3.0:1761315907.292453:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607680 00000100:00100000:3.0:1761315907.292477:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1056 00000100:00100000:3.0:1761315907.292480:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322607680:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315907.292496:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315907.292524:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322607680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 43us (587us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315907.292530:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1056 00000100:00100000:3.0:1761315907.292545:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322607616 00000100:00100000:3.0:1761315907.292561:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1812 00000100:00100000:3.0:1761315907.292564:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322607616:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292584:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322607616:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (850us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315907.292589:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1812 00000100:00100000:3.0:1761315907.292624:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322607744:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292632:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2415dea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322608128:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315907.292642:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322608384:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315907.293447:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608064 00000100:00100000:0.0:1761315907.293481:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2426 00000100:00100000:0.0:1761315907.293485:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322608000:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315907.293571:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315907.293618:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322608000:12345-0@lo:8:kworker/u8:2.0 Request processed in 132us (1934us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315907.293624:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2426 00000100:00100000:2.0:1761315907.293668:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322608192:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293677:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ad8b822 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322607808:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293695:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322607616:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293702:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322607872:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293738:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608256 00000100:00100000:2.0:1761315907.293759:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1599 00000100:00100000:2.0:1761315907.293763:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322608256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293791:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322608256:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (1974us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315907.293796:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1599 00000100:00100000:2.0:1761315907.293804:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2427 00000100:00100000:2.0:1761315907.293807:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322608064:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315907.293875:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315907.293892:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322608064:12345-0@lo:8:kworker/u8:2.0 Request processed in 84us (2120us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315907.293897:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2427 00000100:00100000:2.0:1761315907.293912:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315907.293936:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322607680:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315907.293944:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322607936:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.293959:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608320 00000100:00100000:2.0:1761315907.293972:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1600 00000100:00100000:2.0:1761315907.293976:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322608320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.294005:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322608320:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (94us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315907.294011:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1600 00000100:00100000:2.0:1761315907.294028:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000005042c53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322608320:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315907.294043:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322608256:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315907.294085:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315907.294088:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315907.294091:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322608000:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315907.294097:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315907.294099:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315907.294101:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000515354f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322608064:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315907.295098:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315908.315553:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315909.326605:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315909.326651:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315909.326684:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315909.326723:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315909.326803:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572288 00000100:00100000:2.0:1761315909.326860:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572352 00000100:00100000:2.0:1761315909.326878:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4933 00000100:00100000:2.0:1761315909.326884:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bae14e16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307572288:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315909.326921:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4934 00000100:00100000:1.0:1761315909.326930:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005167c36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307572352:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315909.326944:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bae14e16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307572288:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 59us (345us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315909.326952:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4933 00000100:00100000:1.0:1761315909.326971:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005167c36d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307572352:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 40us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315909.326979:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4934 00000100:00100000:2.0:1761315909.326992:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572416 00000100:00100000:2.0:1761315909.327014:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2428 00000100:00100000:2.0:1761315909.327019:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307572416:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315909.327117:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315909.327143:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307572416:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 124us (493us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315909.327153:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2428 00000100:00100000:3.0:1761315909.327218:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572224 00000100:00100000:3.0:1761315909.327250:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1813 00000100:00100000:3.0:1761315909.327254:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307572224:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315909.327276:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307572224:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 21us (592us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315909.327282:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1813 00010000:00010000:0.0:1761315909.342859:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315909.851626:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322608448:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315909.851704:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315909.853304:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608448 00000100:00100000:0.0:1761315909.853371:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3081 00000100:00100000:0.0:1761315909.853377:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322608448:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315909.853412:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315909.854298:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322608448:12345-0@lo:41:osp-pre-1-0.0 Request processed in 917us (2601us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315909.854308:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3081 00000100:00100000:2.0:1761315909.855665:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000022eb81a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322608448:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315910.363517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315911.390853:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315912.411453:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315912.412247:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315912.412272:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322608512:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315912.412369:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315912.412458:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315912.412462:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315912.412465:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315912.412481:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4154 00000100:00080000:1.0:1761315912.412484:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315912.412492:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315912.412515:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060be72a7 x1846869322608832/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315912.412531:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315912.412535:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315912.412537:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315912.412539:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4154 00000100:00080000:1.0:1761315912.412541:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315912.412544:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315912.412555:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000048f7066f x1846869322608896/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315912.412732:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322608576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.412764:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.412779:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322608960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.412800:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.412815:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315912.412820:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322608640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.412839:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.412853:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322609024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.412879:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.412910:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608576 00000100:00100000:1.0:1761315912.412946:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1814 00000100:00100000:1.0:1761315912.412951:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322608576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.412990:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322608576:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.412996:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1814 00000100:00100000:1.0:1761315912.413027:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608960 00000100:00100000:1.0:1761315912.413050:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608640 00000100:00100000:1.0:1761315912.413066:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609024 00000100:00100000:1.0:1761315912.413092:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4935 00000100:00100000:1.0:1761315912.413096:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c91750f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322608960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413129:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c91750f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322608960:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413135:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4935 00000100:00100000:1.0:1761315912.413146:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4936 00000100:00100000:1.0:1761315912.413149:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000c58668 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322608640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413173:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000c58668 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322608640:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413179:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4936 00000100:00100000:1.0:1761315912.413188:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4937 00000100:00100000:1.0:1761315912.413191:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002deaa2ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322609024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413213:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002deaa2ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322609024:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413218:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4937 00000100:00100000:1.0:1761315912.413253:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322608704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413274:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.413288:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004486994d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413325:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.413369:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315912.413375:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322608768:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413400:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.413415:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322609152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413435:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.413462:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609088 00000100:00100000:1.0:1761315912.413484:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609152 00000100:00100000:1.0:1761315912.413501:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1601 00000100:00100000:1.0:1761315912.413504:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322609088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413538:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322609088:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413544:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1601 00000100:00100000:1.0:1761315912.413555:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1602 00000100:00100000:1.0:1761315912.413558:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322609152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413585:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322609152:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413590:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1602 00000100:00100000:1.0:1761315912.413622:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608512 00000100:00100000:1.0:1761315912.413646:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608768 00000100:00100000:1.0:1761315912.413666:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3082 00000100:00100000:1.0:1761315912.413670:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322608512:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315912.413704:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315912.413731:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322608512:12345-0@lo:41:osp-pre-0-1.0 Request processed in 60us (1364us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413737:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3082 00000100:00100000:1.0:1761315912.413748:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3083 00000100:00100000:1.0:1761315912.413751:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322608768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413782:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322608768:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413787:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3083 00000100:00100000:1.0:1761315912.413809:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608704 00000100:00100000:1.0:1761315912.413827:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4938 00000100:00100000:1.0:1761315912.413830:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2541175 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322608704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413859:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2541175 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322608704:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (585us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.413864:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4938 00000100:00100000:1.0:1761315912.413897:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003588202b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322608640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413906:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322609024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413914:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9b923fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322608576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413923:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322608960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413955:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322608768:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413963:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322609152:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413970:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322608704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413977:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004486994d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315912.413988:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322608512:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315912.414038:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322608832:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315912.414063:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.414075:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322608896:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315912.414091:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.414137:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608832 00000100:00100000:1.0:1761315912.414156:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322608896 00000100:00100000:1.0:1761315912.414170:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2429 00000100:00100000:1.0:1761315912.414174:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322608832:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315912.414270:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315912.414299:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322608832:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (236us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315912.414305:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2429 00000100:00100000:1.0:1761315912.414314:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2430 00000100:00100000:1.0:1761315912.414317:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322608896:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315912.414416:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315912.414440:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322608896:12345-0@lo:8:kworker/u8:2.0 Request processed in 121us (348us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315912.414445:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2430 00000100:00080000:1.0:1761315912.414468:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315912.414472:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315912.414475:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322608832:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315912.414482:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315912.414484:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315912.414486:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000048f7066f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322608896:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315912.414507:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322609216:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315912.414530:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315912.414567:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609216 00000100:00100000:1.0:1761315912.414589:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1057 00000100:00100000:1.0:1761315912.414592:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322609216:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315912.414610:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315912.414640:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322609216:12345-0@lo:13:osp-pre-1-1.0 Request processed in 47us (110us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315912.414646:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1057 00000100:00100000:1.0:1761315912.414691:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322609216:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315913.435522:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315914.441527:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315914.441576:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315914.441608:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315914.441646:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572544 00000100:00100000:1.0:1761315914.441648:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572480 00000100:00100000:0.0:1761315914.441714:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572608 00000100:00100000:0.0:1761315914.441736:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4939 00000100:00100000:0.0:1761315914.441743:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307572544:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315914.441804:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000022eb81a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307572544:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 62us (286us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315914.441811:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4939 00000100:00100000:0.0:1761315914.441826:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4940 00000100:00100000:0.0:1761315914.441830:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307572608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315914.441861:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000005042c53 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307572608:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 30us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315914.441867:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4940 00000100:00100000:3.0:1761315914.442291:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1815 00000100:00100000:3.0:1761315914.442297:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307572480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315914.442372:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307572480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 46us (767us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315914.442379:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1815 00000100:00100000:3.0:1761315914.442897:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315914.443075:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572672 00000100:00100000:2.0:1761315914.443787:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2431 00000100:00100000:2.0:1761315914.443794:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307572672:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315914.443886:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315914.443950:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307572672:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 155us (1054us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315914.443958:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2431 00010000:00010000:1.0:1761315914.460098:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315914.971577:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315914.971601:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322609280:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315914.971680:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315914.971786:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609280 00000100:00100000:3.0:1761315914.971845:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3084 00000100:00100000:3.0:1761315914.971852:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322609280:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315914.971884:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315914.971935:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322609280:12345-0@lo:41:osp-pre-1-0.0 Request processed in 82us (255us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315914.971942:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3084 00000100:00100000:2.0:1761315914.972271:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c18bc7a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322609280:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315915.483556:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315916.507493:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315917.531550:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609344:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315917.531617:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.532039:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609344 00000100:00100000:1.0:1761315917.532092:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1058 00000100:00100000:3.0:1761315917.532093:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315917.532100:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322609344:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315917.532103:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609408:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315917.532140:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315917.532170:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609408 00000100:00100000:3.0:1761315917.532200:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3085 00000100:00100000:3.0:1761315917.532204:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322609408:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315917.532229:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315917.532268:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322609408:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (129us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315917.532276:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3085 00000100:00100000:3.0:1761315917.532317:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609408:0@lo:41:osp-pre-0-1.0 00002000:00000024:1.0:1761315917.532322:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315917.532404:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322609344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 303us (787us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315917.532413:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1058 00000100:00100000:3.0:1761315917.532416:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322609344:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315917.534956:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315917.534962:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315917.534966:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315917.534972:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4159 00000100:00080000:0.0:1761315917.534976:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315917.534982:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315917.535001:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f0be819d x1846869322609728/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315917.535015:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315917.535018:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315917.535020:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315917.535023:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4159 00000100:00080000:0.0:1761315917.535033:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315917.535037:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315917.535048:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c77a9413 x1846869322609792/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315917.535089:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322609664:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535142:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315917.535168:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315917.535173:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322609600:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535196:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315917.535212:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322609984:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535233:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315917.535266:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609600 00000100:00100000:0.0:1761315917.535304:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4941 00000100:00100000:0.0:1761315917.535310:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a476cb8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322609600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535372:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a476cb8a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322609600:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (176us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315917.535380:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4941 00000100:00100000:0.0:1761315917.535403:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609984 00000100:00100000:0.0:1761315917.535426:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1603 00000100:00100000:0.0:1761315917.535429:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322609984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535456:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322609984:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315917.535462:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1603 00000100:00100000:0.0:1761315917.535526:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322609600:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.535536:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322609984:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535721:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322609536:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535752:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.535778:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322609920:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535798:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.535815:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315917.535821:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322609472:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535844:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.535857:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322609856:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535875:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.535895:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322609728:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535916:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315917.535928:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322609792:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315917.535945:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1761315917.535966:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315917.535976:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609472 00000100:00100000:1.0:1761315917.535997:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1816 00000100:00100000:1.0:1761315917.536002:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322609472:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.536032:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322609472:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (187us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315917.536037:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1816 00000100:00100000:1.0:1761315917.536063:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609536 00000100:00100000:1.0:1761315917.536082:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609920 00000100:00100000:1.0:1761315917.536095:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609856 00000100:00100000:1.0:1761315917.536109:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4942 00000100:00100000:1.0:1761315917.536113:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322609536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.536141:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000689c5526 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322609536:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (389us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315917.536147:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4942 00000100:00100000:1.0:1761315917.536157:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4943 00000100:00100000:1.0:1761315917.536160:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322609920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.536184:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604dcad7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322609920:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (386us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315917.536189:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4943 00000100:00100000:1.0:1761315917.536199:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4944 00000100:00100000:1.0:1761315917.536202:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322609856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315917.536225:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a2ab8d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322609856:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (349us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315917.536231:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4944 00000100:00100000:1.0:1761315917.536256:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609728 00000100:00100000:1.0:1761315917.536271:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609792 00000100:00100000:1.0:1761315917.536284:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2432 00000100:00100000:1.0:1761315917.536287:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322609728:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315917.536397:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315917.536431:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322609728:12345-0@lo:8:kworker/u8:0.0 Request processed in 142us (514us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315917.536439:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2432 00000100:00100000:1.0:1761315917.536452:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2433 00000100:00100000:1.0:1761315917.536455:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322609792:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315917.536481:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322609664 00000100:00100000:3.0:1761315917.536519:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3086 00000100:00100000:3.0:1761315917.536524:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322609664:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315917.536536:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315917.536565:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322609792:12345-0@lo:8:kworker/u8:0.0 Request processed in 108us (619us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315917.536570:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2433 00000100:00100000:3.0:1761315917.536587:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322609664:12345-0@lo:400:kworker/u8:0.0 Request processed in 62us (1445us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315917.536596:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3086 00000100:00080000:1.0:1761315917.536605:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315917.536609:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315917.536614:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f0be819d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322609728:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315917.536626:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315917.536629:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315917.536631:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c77a9413 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322609792:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315917.536675:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000081cfb90a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322609472:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315917.536689:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322609856:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315917.536698:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008bb7a9f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322609536:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315917.536714:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322609920:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315917.536842:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aff84b27 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322609664:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315918.555523:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315919.560077:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315919.560128:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315919.560172:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315919.560201:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315919.560203:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572864 00000100:00100000:3.0:1761315919.560228:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572800 00000100:00100000:3.0:1761315919.560288:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4945 00000100:00100000:3.0:1761315919.560302:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307572800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315919.560319:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4946 00000100:00100000:1.0:1761315919.560325:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c18bc7a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307572864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315919.560496:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572928 00000100:00100000:0.0:1761315919.560520:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2434 00000100:00100000:0.0:1761315919.560525:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307572928:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761315919.560616:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315919.560750:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572736 00000100:00100000:2.0:1761315919.560781:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1817 00000100:00100000:2.0:1761315919.560785:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307572736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315919.560922:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307572736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 136us (795us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315919.560922:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1516f67 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307572800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 618us (718us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315919.560931:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1817 00000100:00100000:3.0:1761315919.560931:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4945 00000100:00100000:0.0:1761315919.560936:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307572928:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 407us (865us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315919.560945:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2434 00000100:00100000:1.0:1761315919.561126:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c18bc7a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307572864:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 799us (954us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315919.561138:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4946 00010000:00010000:0.0:1761315919.580839:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315920.094931:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761315920.094951:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322610048:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315920.095016:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315920.095150:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610048 00000100:00100000:3.0:1761315920.099104:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3087 00000100:00100000:3.0:1761315920.099115:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322610048:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315920.099158:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315920.099239:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322610048:12345-0@lo:41:osp-pre-1-0.0 Request processed in 125us (4224us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315920.099245:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3087 00000100:00100000:2.0:1761315920.099308:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322610048:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315920.604743:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315921.627506:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315922.651873:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315922.654910:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315922.654935:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322610176:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315922.655008:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655049:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322610112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315922.655073:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315922.655102:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610176 00000100:00100000:1.0:1761315922.655103:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610112 00000100:00100000:3.0:1761315922.655148:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3088 00000100:00100000:1.0:1761315922.655152:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1059 00000100:00100000:3.0:1761315922.655154:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322610176:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315922.655155:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322610112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315922.655181:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000004:00000024:3.0:1761315922.655181:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315922.655217:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322610176:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (210us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315922.655224:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3088 00000100:00100000:1.0:1761315922.655420:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322610112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 263us (346us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315922.655427:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1059 00000100:00080000:2.0:1761315922.655443:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315922.655446:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ac61232 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322610240:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315922.655461:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315922.655465:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315922.655470:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4165 00000100:00080000:2.0:1761315922.655472:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:1.0:1761315922.655474:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315922.655475:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315922.655494:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000042bf6d90 x1846869322610496/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315922.655502:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322610176:0@lo:41:osp-pre-0-1.0 00000100:00080000:2.0:1761315922.655507:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315922.655510:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315922.655512:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315922.655514:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4165 00000100:00100000:1.0:1761315922.655515:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322610112:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1761315922.655516:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315922.655518:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315922.655527:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006b1d3743 x1846869322610560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315922.655532:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315922.655536:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322610432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655556:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655569:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322610624:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655589:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655624:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000042bf6d90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322610496:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655650:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655664:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322610560:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655683:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655709:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610624 00000100:00100000:2.0:1761315922.655709:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322610368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655737:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4947 00000100:00100000:2.0:1761315922.655737:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655742:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322610624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315922.655754:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610240 00000100:00100000:2.0:1761315922.655755:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322610752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655774:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048f7066f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322610624:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (184us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315922.655779:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655781:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4947 00000100:00100000:1.0:1761315922.655799:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610368 00000100:00100000:2.0:1761315922.655801:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761315922.655802:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1818 00000100:00100000:2.0:1761315922.655807:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322610304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315922.655809:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322610240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.655821:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4948 00000100:00100000:1.0:1761315922.655825:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322610368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315922.655842:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.655995:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b600536 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322610368:12345-0@lo:400:kworker/u8:0.0 Request processed in 168us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315922.656001:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4948 00000100:00100000:1.0:1761315922.656012:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610304 00000100:00100000:3.0:1761315922.656027:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610432 00000100:00100000:1.0:1761315922.656029:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4949 00000100:00100000:1.0:1761315922.656032:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322610304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315922.656048:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3089 00000100:00100000:3.0:1761315922.656052:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322610432:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656056:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a8ab968 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322610304:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (215us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315922.656062:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4949 00000100:00100000:3.0:1761315922.656078:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322610432:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (521us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315922.656083:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3089 00000100:00100000:1.0:1761315922.656085:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322610624:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656093:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322610432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656101:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ac61232 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322610240:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656130:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610496 00000100:00100000:1.0:1761315922.656145:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610560 00000100:00100000:1.0:1761315922.656156:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610752 00000100:00100000:1.0:1761315922.656172:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1604 00000100:00100000:1.0:1761315922.656176:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322610752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656204:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322610752:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (426us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315922.656204:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2435 00000100:00100000:3.0:1761315922.656208:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009ce0dc46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322610496:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315922.656210:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1604 00000100:00100000:1.0:1761315922.656218:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2436 00000100:00100000:1.0:1761315922.656220:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322610560:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315922.656395:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761315922.656396:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315922.656425:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322610560:12345-0@lo:8:kworker/u8:0.0 Request processed in 203us (741us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315922.656430:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009ce0dc46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322610496:12345-0@lo:8:kworker/u8:0.0 Request processed in 222us (780us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315922.656432:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2436 00000100:00100000:3.0:1761315922.656436:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2435 00000100:00080000:1.0:1761315922.656466:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315922.656470:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315922.656472:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000042bf6d90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322610496:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315922.656480:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315922.656482:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315922.656484:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b1d3743 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322610560:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315922.656877:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000058d40a7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322610304:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315922.656887:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322610688:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315922.656916:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315922.658834:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610688 00000100:00100000:1.0:1761315922.658899:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4950 00000100:00100000:1.0:1761315922.658905:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322610688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315922.658987:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024202054 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322610688:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (2070us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315922.658994:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4950 00000100:00100000:2.0:1761315922.659128:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000038772896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322610688:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315922.659143:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322610368:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315922.659154:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322610752:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315922.659386:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322610240:12345-0@lo:400:kworker/u8:0.0 Request processed in 3576us (3909us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315922.659396:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1818 00010000:00010000:2.0:1761315923.675796:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315924.681749:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315924.681801:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315924.681836:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315924.681862:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315924.681964:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307572992 00000100:00100000:0.0:1761315924.682018:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1819 00000100:00100000:0.0:1761315924.682024:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307572992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315924.682085:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307572992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 60us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315924.682093:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1819 00000100:00100000:1.0:1761315924.682315:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573120 00000100:00100000:1.0:1761315924.682393:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573056 00000100:00100000:1.0:1761315924.682413:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4951 00000100:00100000:1.0:1761315924.682418:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307573120:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315924.682453:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307573120:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 35us (617us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315924.682459:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4951 00000100:00100000:1.0:1761315924.682472:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4952 00000100:00100000:1.0:1761315924.682476:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307573056:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315924.682500:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db207c22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307573056:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 23us (637us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315924.682505:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4952 00000100:00100000:2.0:1761315924.682625:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573184 00000100:00100000:2.0:1761315924.682657:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2437 00000100:00100000:2.0:1761315924.682661:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307573184:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315924.682753:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315924.682779:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307573184:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 117us (1038us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315924.682785:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2437 00010000:00010000:0.0:1761315924.699540:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315925.212499:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315925.213376:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322610816:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315925.213445:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315925.213547:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610816 00000100:00100000:0.0:1761315925.213612:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3090 00000100:00100000:0.0:1761315925.213618:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322610816:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315925.213652:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315925.213691:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322610816:12345-0@lo:41:osp-pre-1-0.0 Request processed in 73us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315925.213698:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3090 00000100:00100000:1.0:1761315925.213787:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322610816:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315925.723569:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315926.747606:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315927.771683:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315927.771706:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322610880:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315927.771773:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315927.771957:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315927.771961:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315927.771964:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315927.771969:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4170 00000100:00080000:3.0:1761315927.771972:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315927.771977:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315927.771996:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000013d31559 x1846869322611264/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315927.772008:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315927.772011:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315927.772012:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315927.772015:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4170 00000100:00080000:3.0:1761315927.772016:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315927.772018:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315927.772027:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000af1aef25 x1846869322611328/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315927.772062:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322611136:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315927.772093:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315927.772108:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000055d37d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322611520:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315927.772128:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315927.772141:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315927.772145:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322611072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315927.772177:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315927.772188:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322611456:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315927.772205:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315927.772267:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322610944:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.772301:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315927.772325:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000091728e30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322611200:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.772365:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315927.772383:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315927.772388:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322611008:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.772501:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315927.772516:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322611392:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.772538:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315927.772990:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322611264:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315927.773757:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610944 00000100:00100000:0.0:1761315927.773990:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322610880 00000100:00100000:0.0:1761315927.774021:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1060 00000100:00100000:0.0:1761315927.774026:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322610880:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315927.774054:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315927.774070:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1820 00000100:00100000:3.0:1761315927.774076:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322610944:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774108:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322610880:12345-0@lo:13:osp-pre-1-1.0 Request processed in 81us (2337us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774116:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1060 00000100:00100000:3.0:1761315927.774124:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322610944:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (1822us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315927.774133:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1820 00000100:00100000:0.0:1761315927.774146:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611136 00000100:00100000:0.0:1761315927.774176:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611456 00000100:00100000:0.0:1761315927.774194:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611008 00000100:00100000:0.0:1761315927.774204:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611392 00000100:00100000:0.0:1761315927.774216:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4953 00000100:00100000:0.0:1761315927.774220:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eccc01e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322611136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774254:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eccc01e8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322611136:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (2160us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774259:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4953 00000100:00100000:0.0:1761315927.774270:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4954 00000100:00100000:0.0:1761315927.774273:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322611456:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774297:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099fc3140 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322611456:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (2092us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774302:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4954 00000100:00100000:0.0:1761315927.774312:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4955 00000100:00100000:0.0:1761315927.774315:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322611008:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774337:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001883266f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322611008:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (1837us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774360:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4955 00000100:00100000:0.0:1761315927.774372:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4956 00000100:00100000:0.0:1761315927.774375:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322611392:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774400:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ddd915b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322611392:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (1862us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774405:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4956 00000100:00100000:0.0:1761315927.774429:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611072 00000100:00100000:0.0:1761315927.774451:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611200 00000100:00100000:0.0:1761315927.774467:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3091 00000100:00100000:0.0:1761315927.774471:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322611072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774497:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322611072:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (2319us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774502:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3091 00000100:00100000:0.0:1761315927.774511:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3092 00000100:00100000:0.0:1761315927.774515:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322611200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774537:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322611200:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (2194us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774542:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3092 00000100:00100000:0.0:1761315927.774578:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322611584:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315927.774604:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315927.774631:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322611008:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774640:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322611392:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774647:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009df8a936 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322610944:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774655:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000091728e30 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322611200:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315927.774689:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611584 00000100:00100000:0.0:1761315927.774704:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3093 00000100:00100000:0.0:1761315927.774707:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322611584:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315927.774725:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315927.774753:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322611584:12345-0@lo:41:osp-pre-0-1.0 Request processed in 45us (149us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774760:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3093 00000100:00100000:2.0:1761315927.774775:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ceceb46b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322611072:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.774789:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322611456:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315927.774799:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322611136:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315927.774811:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315927.774833:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051ddfaba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322611584:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315927.774872:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611520 00000100:00100000:2.0:1761315927.774912:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1605 00000100:00100000:0.0:1761315927.774917:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6bb4a39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322610880:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315927.774920:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322611520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315927.774928:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315927.774952:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611264 00000100:00100000:1.0:1761315927.774958:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322611328:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315927.774972:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322611520:12345-0@lo:400:kworker/u8:0.0 Request processed in 53us (2842us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315927.774976:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2438 00000100:00100000:0.0:1761315927.774982:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322611264:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315927.774986:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1605 00000100:00100000:1.0:1761315927.774992:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315927.775014:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611328 00000100:00100000:1.0:1761315927.775030:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2439 00000100:00100000:1.0:1761315927.775036:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322611328:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315927.775061:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000055d37d5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322611520:0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761315927.775113:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315927.775160:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322611264:12345-0@lo:8:kworker/u8:0.0 Request processed in 178us (238us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315927.775169:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2438 00010000:02020000:1.0:1761315927.775169:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315927.775202:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322611328:12345-0@lo:8:kworker/u8:0.0 Request processed in 165us (211us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315927.775210:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2439 00000100:00080000:2.0:1761315927.775676:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315927.775681:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315927.775685:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322611264:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761315927.775692:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315927.775695:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315927.775697:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322611328:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315928.795553:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315929.800254:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315929.801082:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315929.801126:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315929.801133:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573440 00000100:00100000:3.0:1761315929.801158:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315929.801176:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573312 00000100:00100000:0.0:1761315929.801179:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2440 00000100:00100000:0.0:1761315929.801187:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307573440:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:1.0:1761315929.801215:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4957 00000100:00100000:1.0:1761315929.801221:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000055d37d5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307573312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00010000:02020000:0.0:1761315929.801283:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315929.801293:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000055d37d5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307573312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 71us (1046us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315929.801302:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4957 00000100:00100000:0.0:1761315929.801318:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307573440:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 129us (237us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315929.801326:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2440 00000100:00100000:2.0:1761315929.801441:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573248 00000100:00100000:2.0:1761315929.801482:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1821 00000100:00100000:2.0:1761315929.801487:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307573248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315929.801535:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307573248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 48us (411us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315929.801542:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1821 00000100:00100000:2.0:1761315929.801690:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573376 00000100:00100000:2.0:1761315929.801718:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4958 00000100:00100000:2.0:1761315929.801721:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307573376:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315929.801757:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d4f7a52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307573376:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 35us (601us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315929.801763:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4958 00010000:00010000:1.0:1761315929.819769:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315930.332528:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322611648:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315930.332605:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315930.332715:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611648 00000100:00100000:2.0:1761315930.332768:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3094 00000100:00100000:2.0:1761315930.332775:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322611648:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315930.332809:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315930.332865:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322611648:12345-0@lo:41:osp-pre-1-0.0 Request processed in 89us (261us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315930.332881:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3094 00000100:00100000:3.0:1761315930.332973:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322611648:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315930.843506:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315931.867472:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315932.891573:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315932.891641:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322611712:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315932.891710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315932.891807:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611712 00000100:00100000:0.0:1761315932.891859:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3095 00000100:00100000:0.0:1761315932.891866:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322611712:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315932.891904:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315932.891949:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322611712:12345-0@lo:41:osp-pre-0-1.0 Request processed in 83us (240us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315932.891956:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3095 00000100:00100000:2.0:1761315932.892043:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322611712:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315932.892064:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315932.892072:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322611776:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315932.892097:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315932.892219:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611776 00000100:00100000:1.0:1761315932.892264:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1061 00000100:00100000:1.0:1761315932.892269:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322611776:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315932.892290:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315932.892339:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322611776:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (241us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315932.892366:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1061 00000100:00100000:2.0:1761315932.892378:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322611776:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761315932.893176:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315932.893181:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315932.893184:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315932.893189:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4175 00000100:00080000:0.0:1761315932.893193:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315932.893197:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315932.893214:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000de3a181 x1846869322612096/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761315932.893227:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761315932.893231:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761315932.893232:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761315932.893235:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4175 00000100:00080000:0.0:1761315932.893237:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761315932.893239:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315932.893249:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000024a427eb x1846869322612160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315932.893307:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c0153f06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322611840:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.893369:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322612032:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315932.893380:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315932.893400:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322612224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315932.893408:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322612096:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315932.893417:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315932.893444:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315932.893446:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761315932.893455:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315932.893455:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322611968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315932.893472:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315932.893479:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322611904:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315932.893522:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611840 00000100:00100000:0.0:1761315932.893565:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1822 00000100:00100000:0.0:1761315932.893569:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322611840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.894650:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315932.894679:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322612352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.894719:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612224 00000100:00100000:3.0:1761315932.894754:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611968 00000100:00100000:3.0:1761315932.894772:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4959 00000100:00100000:3.0:1761315932.894776:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322612224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.894816:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038772896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322612224:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (1374us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315932.894823:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4959 00000100:00100000:3.0:1761315932.894840:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4960 00000100:00100000:3.0:1761315932.894843:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322611968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.894871:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060c89ebd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322611968:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (223us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315932.894877:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4960 00000100:00100000:3.0:1761315932.894901:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612032 00000100:00100000:3.0:1761315932.894924:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3096 00000100:00100000:3.0:1761315932.894928:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322612032:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.894955:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322612032:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (1543us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315932.894960:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3096 00000100:00100000:3.0:1761315932.895003:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612096 00000100:00100000:3.0:1761315932.895025:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2441 00000100:00100000:3.0:1761315932.895029:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612096:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315932.895116:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315932.895147:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612096:12345-0@lo:8:kworker/u8:0.0 Request processed in 117us (1695us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315932.895153:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2441 00000100:00080000:2.0:1761315932.896015:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315932.896021:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315932.896026:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322612096:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315932.896035:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322612160:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315932.896041:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315932.896071:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322612288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315932.896072:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315932.896106:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315932.896122:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612160 00000100:00100000:1.0:1761315932.896137:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322612224:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315932.896151:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2442 00000100:00100000:2.0:1761315932.896157:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322612160:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315932.896170:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322611904 00000100:00100000:1.0:1761315932.896199:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612288 00000100:00100000:1.0:1761315932.896218:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4961 00000100:00100000:1.0:1761315932.896222:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322611904:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315932.896239:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315932.896266:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058d40a7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322611904:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315932.896274:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4961 00000100:00100000:2.0:1761315932.896284:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322612160:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (212us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315932.896293:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4962 00000100:00100000:2.0:1761315932.896293:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2442 00000100:00100000:1.0:1761315932.896301:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322612288:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761315932.896326:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315932.896330:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315932.896334:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a427eb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322612160:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315932.896338:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b1d3743 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322612288:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (233us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315932.896367:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4962 00000100:00100000:1.0:1761315932.896398:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8a536d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322611904:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315932.896409:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d31559 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322612288:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315932.896486:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322611840:12345-0@lo:400:kworker/u8:0.0 Request processed in 2914us (3107us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315932.896495:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1822 00000100:00100000:1.0:1761315932.896592:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c0153f06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322611840:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.896656:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315932.896691:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322612032:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315932.896708:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008517e317 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322611968:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315932.897063:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612352 00000100:00100000:2.0:1761315932.897249:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1606 00000100:00100000:2.0:1761315932.897253:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322612352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315932.898061:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322612352:12345-0@lo:400:kworker/u8:0.0 Request processed in 805us (1404us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315932.898069:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1606 00000100:00100000:0.0:1761315932.898071:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322612352:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315933.915656:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315934.921980:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315934.922028:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315934.922952:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573504 00000100:00100000:0.0:1761315934.923008:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1823 00000100:00100000:0.0:1761315934.923016:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307573504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315934.923097:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307573504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 81us (1126us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315934.923106:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1823 00000100:00100000:1.0:1761315934.923641:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315934.923733:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573632 00000100:00100000:2.0:1761315934.923769:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4963 00000100:00100000:2.0:1761315934.923774:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042bf6d90 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307573632:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315934.923815:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042bf6d90 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307573632:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 40us (1788us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315934.923822:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4963 00000100:00100000:1.0:1761315934.923985:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315934.924053:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573568 00000100:00100000:2.0:1761315934.924086:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4964 00000100:00100000:2.0:1761315934.924090:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ac61232 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307573568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315934.924126:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ac61232 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307573568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (487us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315934.924132:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4964 00000100:00100000:3.0:1761315934.924186:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573696 00000100:00100000:3.0:1761315934.924251:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2443 00000100:00100000:3.0:1761315934.924255:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307573696:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761315934.924367:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315934.924397:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307573696:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 140us (413us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315934.924403:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2443 00010000:00010000:0.0:1761315934.941225:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315935.453872:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761315935.453896:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f89f3b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322612416:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315935.453971:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315935.454066:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612416 00000100:00100000:3.0:1761315935.454119:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3097 00000100:00100000:3.0:1761315935.454125:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322612416:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315935.454158:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315935.454212:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322612416:12345-0@lo:41:osp-pre-1-0.0 Request processed in 86us (241us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315935.454219:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3097 00000100:00100000:1.0:1761315935.454302:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f89f3b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322612416:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315935.963537:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315936.987575:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315938.011589:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315938.011606:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322612480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315938.011671:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315938.011762:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612480 00000100:00100000:1.0:1761315938.011817:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1062 00000100:00100000:1.0:1761315938.011823:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322612480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315938.011849:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315938.011895:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322612480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (224us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315938.011902:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1062 00000100:00100000:1.0:1761315938.011944:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322612480:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315938.013967:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322612544:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315938.014039:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315938.014103:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612544 00000100:00100000:2.0:1761315938.014150:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3098 00000100:00100000:2.0:1761315938.014156:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322612544:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315938.014186:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315938.014579:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322612544:12345-0@lo:41:osp-pre-0-1.0 Request processed in 421us (540us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315938.014586:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3098 00000100:00100000:3.0:1761315938.014608:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322612544:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315938.014716:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315938.014752:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315938.014753:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322612672:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315938.014757:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315938.014760:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315938.014764:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4180 00000100:00080000:1.0:1761315938.014766:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315938.014770:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761315938.014781:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315938.014786:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000011e09f52 x1846869322612864/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315938.014798:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761315938.014800:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1761315938.014801:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315938.014803:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761315938.014805:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322612608:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315938.014806:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4180 00000100:00080000:1.0:1761315938.014808:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315938.014811:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315938.014819:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000060be72a7 x1846869322612928/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761315938.014827:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315938.014857:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322613056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.014876:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612672 00000100:00100000:0.0:1761315938.014878:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315938.014892:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315938.014897:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322612992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.014900:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613056 00000100:00100000:1.0:1761315938.014916:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4965 00000100:00100000:0.0:1761315938.014917:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315938.014920:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322612672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.014938:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612608 00000100:00100000:0.0:1761315938.014960:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1824 00000100:00100000:1.0:1761315938.014961:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091728e30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322612672:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (180us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315938.014963:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322612608:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.014967:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4965 00000100:00100000:1.0:1761315938.014978:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612992 00000100:00100000:0.0:1761315938.014987:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322612608:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (160us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315938.014993:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1824 00000100:00100000:1.0:1761315938.014993:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4966 00000100:00100000:1.0:1761315938.014996:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322613056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015017:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4967 00000100:00100000:1.0:1761315938.015018:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d31559 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322613056:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (140us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315938.015021:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322612992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015023:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4966 00000100:00100000:1.0:1761315938.015042:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba1b858b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322612736:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015043:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322612992:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (126us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315938.015049:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4967 00000100:00100000:1.0:1761315938.015062:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315938.015073:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322612992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015073:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322613120:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015084:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322613056:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015091:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322612608:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015093:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315938.015098:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322612672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015110:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315938.015114:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000001e53b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322612800:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015124:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612736 00000100:00100000:1.0:1761315938.015133:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315938.015141:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4968 00000100:00100000:0.0:1761315938.015145:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9b923fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322612736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015154:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613120 00000100:00100000:0.0:1761315938.015173:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9b923fa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322612736:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (111us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315938.015175:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1607 00000100:00100000:0.0:1761315938.015179:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4968 00000100:00100000:1.0:1761315938.015179:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322613120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315938.015203:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba1b858b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322612736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315938.015205:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322613120:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (111us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315938.015210:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1607 00000100:00100000:0.0:1761315938.015211:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322613120:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315938.015764:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000011e09f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322612864:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315938.015795:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315938.015821:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322612928:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315938.015840:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315938.015881:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612800 00000100:00100000:2.0:1761315938.015917:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3099 00000100:00100000:2.0:1761315938.015921:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322612800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315938.015956:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322612800:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (821us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315938.015961:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3099 00000100:00100000:3.0:1761315938.016029:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612864 00000100:00100000:3.0:1761315938.016058:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322612928 00000100:00100000:3.0:1761315938.016072:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2444 00000100:00100000:3.0:1761315938.016081:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612864:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315938.016168:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315938.016199:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612864:12345-0@lo:8:kworker/u8:2.0 Request processed in 122us (404us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315938.016205:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2444 00000100:00100000:3.0:1761315938.016217:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2445 00000100:00100000:3.0:1761315938.016219:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612928:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315938.016298:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315938.016321:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322612928:12345-0@lo:8:kworker/u8:2.0 Request processed in 100us (481us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315938.016326:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2445 00000100:00080000:2.0:1761315938.016421:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315938.016426:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315938.016430:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000011e09f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322612864:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761315938.016438:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761315938.016441:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761315938.016443:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000060be72a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322612928:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315938.016593:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000001e53b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322612800:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761315939.038680:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315940.042829:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315940.042878:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315940.042921:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315940.042937:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573760 00000100:00100000:0.0:1761315940.042991:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573824 00000100:00100000:1.0:1761315940.043012:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1825 00000100:00100000:1.0:1761315940.043019:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307573760:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315940.043027:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573888 00000100:00100000:0.0:1761315940.043049:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4969 00000100:00100000:0.0:1761315940.043055:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307573824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315940.043087:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307573760:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (208us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315940.043094:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000e80cc2f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307573824:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 38us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315940.043099:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1825 00000100:00100000:0.0:1761315940.043107:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4969 00000100:00100000:0.0:1761315940.043124:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4970 00000100:00100000:0.0:1761315940.043129:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307573888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315940.043156:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357267b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307573888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 26us (236us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315940.043165:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4970 00000100:00100000:0.0:1761315940.044224:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315940.044295:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307573952 00000100:00100000:1.0:1761315940.045053:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2446 00000100:00100000:1.0:1761315940.045059:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307573952:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315940.045148:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315940.045851:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307573952:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 790us (1629us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315940.045862:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2446 00010000:00010000:2.0:1761315940.059892:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315940.571648:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315940.571668:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322613184:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315940.571733:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315940.571834:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613184 00000100:00100000:2.0:1761315940.571884:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3100 00000100:00100000:2.0:1761315940.571889:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322613184:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315940.571920:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315940.571968:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322613184:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315940.571975:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3100 00000100:00100000:0.0:1761315940.572051:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322613184:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315941.083609:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315942.108064:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315943.131624:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315943.131804:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322613248:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315943.131869:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315943.132131:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613248 00000100:00100000:1.0:1761315943.132184:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1063 00000100:00100000:1.0:1761315943.132190:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322613248:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315943.132218:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315943.132273:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322613248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (405us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315943.132284:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1063 00000100:00080000:3.0:1761315943.132438:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315943.132443:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315943.132446:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315943.132450:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4185 00000100:00080000:3.0:1761315943.132452:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315943.132456:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315943.132482:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004ec538ee x1846869322613632/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315943.132499:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315943.132502:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315943.132503:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315943.132507:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4185 00000100:00080000:3.0:1761315943.132508:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315943.132511:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315943.132521:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d8880f6 x1846869322613696/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315943.132527:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322613504:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132556:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322613632:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315943.132557:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315943.132582:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315943.132594:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322613248:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315943.132594:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d8880f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322613696:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315943.132611:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322613888:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132615:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315943.132629:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315943.132642:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761315943.132648:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322613312:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132668:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613632 00000100:00100000:2.0:1761315943.132672:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315943.132685:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322613568:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132686:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613696 00000100:00100000:3.0:1761315943.132699:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613888 00000100:00100000:2.0:1761315943.132704:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315943.132716:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1608 00000100:00100000:3.0:1761315943.132720:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322613888:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315943.132726:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2447 00000100:00100000:2.0:1761315943.132729:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322613632:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132751:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322613888:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (122us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315943.132757:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1608 00000100:00100000:3.0:1761315943.132767:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2448 00000100:00100000:3.0:1761315943.132770:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322613696:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761315943.132851:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761315943.132877:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315943.132878:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322613632:12345-0@lo:8:kworker/u8:2.0 Request processed in 148us (296us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315943.132884:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2447 00000100:00100000:3.0:1761315943.132894:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322613696:12345-0@lo:8:kworker/u8:2.0 Request processed in 123us (279us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315943.132899:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2448 00000100:00100000:3.0:1761315943.132915:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613568 00000100:00100000:2.0:1761315943.132920:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613312 00000100:00100000:2.0:1761315943.132937:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1826 00000100:00100000:3.0:1761315943.132937:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3101 00000100:00100000:2.0:1761315943.132941:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322613312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.132941:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322613568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315943.132965:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322613312:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315943.132965:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322613568:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315943.132971:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1826 00000100:00100000:3.0:1761315943.132971:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3101 00000100:00100000:2.0:1761315943.132991:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613504 00000100:00080000:3.0:1761315943.132993:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315943.132996:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315943.132999:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322613632:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315943.133006:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761315943.133008:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4971 00000100:00080000:3.0:1761315943.133008:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315943.133010:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d8880f6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322613696:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315943.133012:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322613504:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.133032:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322613312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315943.133037:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008517e317 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322613504:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (480us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315943.133040:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095078c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322613568:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315943.133043:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4971 00000100:00100000:3.0:1761315943.133047:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000096cf7252 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322613888:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.133054:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e9b55f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322613504:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133122:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322613440:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133148:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315943.133176:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322613824:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133195:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315943.133220:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761315943.133225:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322613376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133247:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315943.133259:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000166de8db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322613760:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133278:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315943.133577:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613824 00000100:00100000:3.0:1761315943.133608:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613376 00000100:00100000:3.0:1761315943.133623:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613760 00000100:00100000:3.0:1761315943.133646:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4972 00000100:00100000:3.0:1761315943.133650:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322613824:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.133684:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001e53b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322613824:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (488us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315943.133690:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4972 00000100:00100000:3.0:1761315943.133701:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4973 00000100:00100000:3.0:1761315943.133704:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322613376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315943.133714:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613440 00000100:00100000:3.0:1761315943.133730:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060be72a7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322613376:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (483us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315943.133735:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4973 00000100:00100000:2.0:1761315943.133736:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3102 00000100:00100000:2.0:1761315943.133740:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322613440:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.133744:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4974 00000100:00100000:3.0:1761315943.133746:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322613760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315943.133765:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011e09f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322613760:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (487us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315943.133766:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322613440:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (618us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315943.133772:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3102 00000100:00100000:3.0:1761315943.133772:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4974 00000100:00100000:1.0:1761315943.133855:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000496ea1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322613376:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133869:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000166de8db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322613760:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133877:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000740059ce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322613440:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.133884:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322613824:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315943.134114:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322613952:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315943.134143:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315943.134374:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322613952 00000100:00100000:2.0:1761315943.134409:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3103 00000100:00100000:2.0:1761315943.134413:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322613952:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315943.134435:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315943.134483:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322613952:12345-0@lo:41:osp-pre-0-1.0 Request processed in 69us (339us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315943.134488:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3103 00000100:00100000:1.0:1761315943.134555:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322613952:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761315944.155638:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315945.161329:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315945.161412:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315945.161428:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315945.161479:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315945.161555:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574080 00000100:00100000:0.0:1761315945.161604:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574144 00000100:00100000:0.0:1761315945.161620:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4975 00000100:00100000:0.0:1761315945.161626:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307574080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315945.161691:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8a536d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307574080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (369us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315945.161700:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4975 00000100:00100000:0.0:1761315945.161714:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4976 00000100:00100000:0.0:1761315945.161717:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307574144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315945.161735:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a427eb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307574144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 17us (307us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315945.161741:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4976 00000100:00100000:3.0:1761315945.161910:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574016 00000100:00100000:3.0:1761315945.161942:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1827 00000100:00100000:3.0:1761315945.161946:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307574016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761315945.161983:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307574016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (572us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315945.161997:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1827 00000100:00100000:1.0:1761315945.162201:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574208 00000100:00100000:1.0:1761315945.162228:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2449 00000100:00100000:1.0:1761315945.162232:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307574208:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315945.162489:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315945.162865:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307574208:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 629us (1385us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315945.162873:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2449 00010000:00010000:1.0:1761315945.179701:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315945.696502:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322614016:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315945.696567:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315945.696663:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614016 00000100:00100000:2.0:1761315945.696722:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3104 00000100:00100000:2.0:1761315945.696727:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322614016:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315945.696756:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315945.698189:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322614016:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1457us (1618us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315945.698202:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3104 00000100:00100000:0.0:1761315945.698311:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322614016:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315946.205629:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315947.227516:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315948.251542:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322614208:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315948.251576:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315948.251582:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315948.251585:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315948.251590:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4190 00000100:00080000:3.0:1761315948.251593:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315948.251598:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315948.251612:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.251617:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017545a58 x1846869322614464/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315948.251634:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315948.251637:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315948.251639:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315948.251642:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4190 00000100:00080000:3.0:1761315948.251644:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315948.251647:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315948.251648:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761315948.251656:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba1b858b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322614144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315948.251657:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1a5e34c x1846869322614528/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315948.251689:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315948.251707:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322614400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.251711:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017545a58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322614464:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315948.251735:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.251750:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315948.251757:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322614592:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.251770:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1a5e34c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322614528:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315948.251785:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.251797:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315948.251803:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761315948.251811:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322614784:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.251834:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315948.251864:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614208 00000100:00100000:1.0:1761315948.251896:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614592 00000100:00100000:1.0:1761315948.251910:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4977 00000100:00100000:1.0:1761315948.251915:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322614208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.251953:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ca69bbd1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322614208:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315948.251959:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4977 00000100:00100000:1.0:1761315948.251973:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4978 00000100:00100000:1.0:1761315948.251977:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322614592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.251997:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b1424fc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322614592:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315948.252001:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4978 00000100:00100000:1.0:1761315948.252016:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614144 00000100:00100000:1.0:1761315948.252033:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1064 00000100:00100000:1.0:1761315948.252036:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322614144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315948.252053:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315948.252075:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322614144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 38us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315948.252080:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1064 00000100:00100000:1.0:1761315948.252096:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614464 00000100:00100000:1.0:1761315948.252114:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614528 00000100:00100000:1.0:1761315948.252123:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614784 00000100:00100000:1.0:1761315948.252137:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1609 00000100:00100000:1.0:1761315948.252141:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322614784:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.252164:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322614784:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315948.252169:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1609 00000100:00100000:1.0:1761315948.252177:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2450 00000100:00100000:1.0:1761315948.252180:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322614464:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315948.252276:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315948.252301:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322614464:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (554us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315948.252306:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2450 00000100:00100000:1.0:1761315948.252315:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2451 00000100:00100000:1.0:1761315948.252318:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322614528:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315948.252413:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315948.252444:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322614528:12345-0@lo:8:kworker/u8:2.0 Request processed in 125us (649us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315948.252450:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2451 00000100:00100000:1.0:1761315948.252481:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322614784:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.252489:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322614592:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.252500:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba1b858b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322614144:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315948.252508:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322614208:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315948.254407:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761315948.254557:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315948.254564:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315948.254570:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017545a58 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322614464:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315948.254576:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:3.0:1761315948.254583:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315948.254587:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315948.254590:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1a5e34c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322614528:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315948.254594:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322614848:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315948.254641:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322614080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315948.254647:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.254684:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.254709:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d75a9b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322614336:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315948.254732:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614400 00000100:00100000:3.0:1761315948.254738:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.254757:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322614720:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315948.254770:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614848 00000100:00100000:3.0:1761315948.254784:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315948.254789:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3105 00000100:00100000:2.0:1761315948.254795:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322614400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.254890:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761315948.254896:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322614272:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315948.256114:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3106 00000100:00100000:0.0:1761315948.256124:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322614848:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761315948.256157:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761315948.256210:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322614848:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (1565us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315948.256217:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3106 00000100:00100000:0.0:1761315948.256281:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322614848:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315948.256308:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614080 00000100:00100000:1.0:1761315948.256368:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1828 00000100:00100000:0.0:1761315948.256371:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614336 00000100:00100000:1.0:1761315948.256374:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322614080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315948.256404:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4979 00000100:00100000:0.0:1761315948.256408:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322614336:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315948.257552:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000496ea1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322614336:12345-0@lo:400:kworker/u8:2.0 Request processed in 1139us (2811us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315948.257563:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4979 00000100:00100000:0.0:1761315948.257596:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614720 00000100:00100000:0.0:1761315948.257814:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1610 00000100:00100000:0.0:1761315948.257818:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322614720:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315948.257857:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322614720:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (3075us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315948.257863:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1610 00000100:00100000:2.0:1761315948.258038:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322614400:12345-0@lo:400:kworker/u8:2.0 Request processed in 3241us (6303us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315948.258047:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3105 00000100:00100000:3.0:1761315948.258104:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.258130:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322614656:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.258156:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315948.258178:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322614080:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.258189:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d75a9b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322614336:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.258197:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322614720:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.258225:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614656 00000100:00100000:3.0:1761315948.258246:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4980 00000100:00100000:3.0:1761315948.258250:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000095078c5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322614656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.258278:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000095078c5a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322614656:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (121us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315948.258283:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4980 00000100:00100000:3.0:1761315948.258305:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322614656:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315948.258370:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614272 00000100:00100000:2.0:1761315948.258400:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3107 00000100:00100000:2.0:1761315948.258404:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322614272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315948.258879:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322614272:12345-0@lo:400:kworker/u8:2.0 Request processed in 474us (775us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315948.258887:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3107 00000100:00100000:0.0:1761315948.258967:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322614400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315948.259034:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322614272:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315948.259102:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322614080:12345-0@lo:400:kworker/u8:2.0 Request processed in 2727us (4419us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315948.259111:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1828 00010000:00010000:2.0:1761315949.275521:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315950.279937:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315950.279982:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315950.280018:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315950.280127:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574400 00000100:00100000:2.0:1761315950.280182:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574336 00000100:00100000:2.0:1761315950.280204:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4981 00000100:00100000:2.0:1761315950.280210:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307574400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315950.280278:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307574400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 67us (295us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315950.280287:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4981 00000100:00100000:2.0:1761315950.280303:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4982 00000100:00100000:2.0:1761315950.280307:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307574336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315950.280329:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ceceb46b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307574336:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315950.280334:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4982 00000100:00100000:1.0:1761315950.280461:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574272 00000100:00100000:1.0:1761315950.280500:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1829 00000100:00100000:1.0:1761315950.280504:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307574272:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315950.280531:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307574272:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 26us (602us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315950.280537:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1829 00000100:00100000:2.0:1761315950.282018:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315950.282096:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574464 00000100:00100000:0.0:1761315950.282134:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2452 00000100:00100000:0.0:1761315950.282139:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307574464:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315950.282230:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315950.282288:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307574464:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 147us (274us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315950.282294:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2452 00010000:00010000:0.0:1761315950.304594:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315950.811701:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315950.811722:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322614912:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315950.811791:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315950.811862:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614912 00000100:00100000:2.0:1761315950.811919:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3108 00000100:00100000:2.0:1761315950.811925:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322614912:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315950.811957:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315950.812009:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322614912:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (219us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315950.812016:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3108 00000100:00100000:1.0:1761315950.812050:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322614912:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315951.323499:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315952.348172:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315953.371565:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761315953.371586:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322615040:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315953.371661:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315953.371687:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007dcca1f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322614976:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315953.371710:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315953.371753:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322614976 00000100:00100000:0.0:1761315953.371793:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1065 00000100:00100000:0.0:1761315953.371798:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322614976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315953.371825:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315953.371870:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322614976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (159us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315953.371876:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1065 00000100:00100000:0.0:1761315953.371929:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007dcca1f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322614976:0@lo:13:osp-pre-1-1.0 00010000:00010000:1.0:1761315953.372040:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761315953.372248:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315953.372253:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315953.372256:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315953.372261:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4195 00000100:00080000:2.0:1761315953.372264:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315953.372268:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315953.372285:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000921eafb5 x1846869322615360/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315953.372296:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315953.372299:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315953.372301:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315953.372303:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4195 00000100:00080000:2.0:1761315953.372305:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315953.372308:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315953.372316:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000049a2c7d7 x1846869322615424/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315953.372453:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322615168:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.372489:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315953.372553:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322615552:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.372573:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315953.372728:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b38f16ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322615104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.372757:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315953.372772:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fa280b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322615488:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.372800:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615104 00000100:00100000:0.0:1761315953.372826:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1830 00000100:00100000:0.0:1761315953.372831:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322615104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.373022:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315953.373043:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615168 00000100:00100000:2.0:1761315953.373072:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615552 00000100:00100000:2.0:1761315953.373089:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4983 00000100:00100000:2.0:1761315953.373093:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322615168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315953.373132:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000065048840 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322615168:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (643us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315953.373138:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4983 00000100:00100000:2.0:1761315953.373151:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4984 00000100:00100000:2.0:1761315953.373155:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322615552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315953.373180:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007dcca1f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322615552:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (607us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315953.373185:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4984 00000100:00100000:2.0:1761315953.373231:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322615232:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.373308:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322615104:12345-0@lo:400:kworker/u8:0.0 Request processed in 476us (551us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315953.373315:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1830 00000100:00100000:0.0:1761315953.373386:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322615168:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.373406:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322615552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315953.373490:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322615296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315953.373521:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315953.373644:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b38f16ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322615104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.373662:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315953.373678:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322615616:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.373702:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315953.373733:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615616 00000100:00100000:1.0:1761315953.373758:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1611 00000100:00100000:1.0:1761315953.373762:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322615616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315953.373795:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322615616:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (93us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315953.373801:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1611 00000100:00100000:1.0:1761315953.373819:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615488 00000100:00100000:1.0:1761315953.373836:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615232 00000100:00100000:0.0:1761315953.374553:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4985 00000100:00100000:0.0:1761315953.374559:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322615488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374608:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba1b858b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322615488:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (1586us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315953.374615:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4985 00000100:00100000:0.0:1761315953.374630:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4986 00000100:00100000:0.0:1761315953.374634:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322615232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374660:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322615232:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (998us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315953.374666:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4986 00000100:00100000:0.0:1761315953.374719:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014f5fd29 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322615232:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374733:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008c8247c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322615616:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374778:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fa280b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322615488:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374808:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322615360:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761315953.374838:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315953.375196:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615040 00000100:00100000:2.0:1761315953.375231:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615296 00000100:00100000:2.0:1761315953.375250:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3109 00000100:00100000:2.0:1761315953.375254:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322615040:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315953.375279:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761315953.375320:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3110 00000100:00100000:1.0:1761315953.375325:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322615296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.378052:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049a2c7d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322615424:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315953.379681:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615360 00000100:00100000:3.0:1761315953.379727:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2453 00000100:00100000:3.0:1761315953.379732:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322615360:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315953.379822:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315953.379892:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322615360:12345-0@lo:8:kworker/u8:0.0 Request processed in 158us (5054us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315953.379901:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2453 00000100:00100000:2.0:1761315953.380982:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322615040:12345-0@lo:41:osp-pre-0-1.0 Request processed in 5723us (9318us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315953.380993:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3109 00000100:00100000:3.0:1761315953.381108:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322615040:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315953.381326:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322615296:12345-0@lo:400:kworker/u8:0.0 Request processed in 5998us (7804us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315953.381334:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3110 00000100:00100000:3.0:1761315953.381426:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322615296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315953.381507:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761315953.381546:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315953.381550:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315953.381554:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322615360:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315953.381612:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615424 00000100:00100000:3.0:1761315953.381647:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2454 00000100:00100000:3.0:1761315953.381651:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322615424:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761315953.381745:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315953.382000:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322615424:12345-0@lo:8:kworker/u8:0.0 Request processed in 347us (494us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315953.382008:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2454 00000100:00080000:1.0:1761315953.382054:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315953.382058:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315953.382060:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049a2c7d7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322615424:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315954.398444:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315955.400230:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315955.400283:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315955.400321:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315955.400430:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574592 00000100:00100000:1.0:1761315955.400484:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574656 00000100:00100000:1.0:1761315955.400503:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4987 00000100:00100000:1.0:1761315955.400509:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aff84b27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307574592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315955.400580:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aff84b27 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307574592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315955.400588:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4987 00000100:00100000:1.0:1761315955.400604:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4988 00000100:00100000:1.0:1761315955.400608:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307574656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315955.400629:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008bb7a9f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307574656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315955.400635:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4988 00000100:00100000:2.0:1761315955.400815:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574528 00000100:00100000:2.0:1761315955.400849:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1831 00000100:00100000:2.0:1761315955.400854:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307574528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315955.400886:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307574528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (604us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315955.400892:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1831 00000100:00100000:0.0:1761315955.402583:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315955.403945:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574720 00000100:00100000:2.0:1761315955.404708:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2455 00000100:00100000:2.0:1761315955.404733:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307574720:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761315955.404879:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315955.404991:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307574720:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 257us (2414us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315955.405010:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2455 00010000:00010000:2.0:1761315955.421495:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315955.936647:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322615680:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315955.936715:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315955.936749:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615680 00000100:00100000:0.0:1761315955.936805:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3111 00000100:00100000:0.0:1761315955.936811:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322615680:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315955.936843:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315955.936892:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322615680:12345-0@lo:41:osp-pre-1-0.0 Request processed in 81us (178us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315955.936899:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3111 00000100:00100000:1.0:1761315955.937013:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322615680:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761315956.444237:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315957.467530:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315958.491508:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315958.491644:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322615744:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315958.491710:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.491765:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615744 00000100:00100000:1.0:1761315958.491826:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1066 00000100:00100000:1.0:1761315958.491832:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322615744:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315958.491862:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315958.491907:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322615744:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315958.491914:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1066 00000100:00080000:3.0:1761315958.491941:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761315958.491943:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322615936:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315958.491948:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315958.491953:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315958.491960:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4200 00000100:00080000:3.0:1761315958.491963:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315958.491967:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315958.491974:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315958.491989:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c96a4a50 x1846869322616128/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315958.491996:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315958.492019:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322615872:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492044:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315958.492078:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315958.492082:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315958.492084:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315958.492088:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4200 00000100:00100000:1.0:1761315958.492089:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615872 00000100:00080000:3.0:1761315958.492090:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315958.492095:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315958.492110:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000eeae700 x1846869322616192/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315958.492121:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4989 00000100:00100000:1.0:1761315958.492126:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322615872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492160:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322615872:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (117us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315958.492166:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4989 00000100:00100000:1.0:1761315958.492185:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322616128:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492210:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492225:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322616192:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492246:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492277:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616128 00000100:00100000:1.0:1761315958.492292:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616192 00000100:00100000:1.0:1761315958.492314:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2456 00000100:00100000:1.0:1761315958.492318:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322616128:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492402:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322616256:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492431:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:02020000:1.0:1761315958.492443:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315958.492453:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761315958.492461:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322616320:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492481:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322616128:12345-0@lo:8:kworker/u8:0.0 Request processed in 161us (270us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315958.492488:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2456 00000100:00100000:3.0:1761315958.492492:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492503:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2457 00000100:00100000:1.0:1761315958.492508:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322616192:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492537:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322616000:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492560:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315958.492574:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322616384:0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315958.492583:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315958.492597:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492611:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322616192:12345-0@lo:8:kworker/u8:0.0 Request processed in 102us (366us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315958.492618:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2457 00000100:00100000:1.0:1761315958.492635:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616384 00000100:00100000:3.0:1761315958.492642:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322615744:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315958.492657:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1612 00000100:00100000:1.0:1761315958.492662:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322616384:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492669:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761315958.492677:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322615808:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492691:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322616384:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (94us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315958.492698:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1612 00000100:00100000:3.0:1761315958.492714:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492724:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616256 00000100:00100000:3.0:1761315958.492734:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322616064:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492747:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616320 00000100:00100000:3.0:1761315958.492761:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315958.492767:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616000 00000100:00100000:3.0:1761315958.492788:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322616384:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492789:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4990 00000100:00100000:1.0:1761315958.492796:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7ad9d26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322616256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.492818:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4991 00000100:00100000:3.0:1761315958.492824:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dbb03c9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322616320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.492849:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7ad9d26 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322616256:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (417us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315958.492857:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4990 00000100:00100000:3.0:1761315958.492862:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dbb03c9d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322616320:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (370us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315958.492870:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4991 00000100:00100000:2.0:1761315958.492972:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322616448:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315958.493001:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315958.493057:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615936 00000100:00100000:2.0:1761315958.493078:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616064 00000100:00100000:2.0:1761315958.493087:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616448 00000100:00100000:2.0:1761315958.493096:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3112 00000100:00100000:2.0:1761315958.493099:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322615936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315958.493128:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322615936:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (1155us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315958.493134:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3112 00000100:00100000:2.0:1761315958.493144:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3113 00000100:00100000:2.0:1761315958.493146:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322616064:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315958.493173:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322616064:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (413us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315958.493178:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3113 00000100:00100000:2.0:1761315958.493188:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3114 00000100:00100000:2.0:1761315958.493191:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322616448:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315958.493211:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315958.493237:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322616448:12345-0@lo:41:osp-pre-0-1.0 Request processed in 44us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315958.493242:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3114 00000100:00100000:2.0:1761315958.493258:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322615808 00000100:00100000:2.0:1761315958.493277:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1832 00000100:00100000:2.0:1761315958.493280:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322615808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315958.493301:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322615808:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (590us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315958.493306:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1832 00000100:00100000:2.0:1761315958.493339:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a87f6aae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322616448:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315958.493392:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322615808:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315958.493399:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322616064:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.495798:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4992 00000100:00100000:1.0:1761315958.495862:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322616000:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.496294:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322616320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.496309:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322616256:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761315958.496398:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315958.496403:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315958.496406:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322616128:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761315958.496416:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315958.496418:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315958.496420:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322616192:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315958.496454:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007440e179 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322615872:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315958.496468:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006142da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322615936:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315958.496642:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093e23d65 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322616000:12345-0@lo:400:kworker/u8:0.0 Request processed in 783us (4081us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315958.496654:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4992 00000100:00100000:3.0:1761315958.497330:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322616000:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761315959.515606:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315960.519682:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315960.519730:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315960.519760:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315960.519790:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574912 00000100:00100000:3.0:1761315960.519837:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574848 00000100:00100000:2.0:1761315960.519842:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315960.519862:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4993 00000100:00100000:3.0:1761315960.519870:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a87f6aae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307574912:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761315960.519874:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574976 00000100:00100000:2.0:1761315960.519901:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2458 00000100:00100000:2.0:1761315960.519906:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307574976:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:3.0:1761315960.519944:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a87f6aae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307574912:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 75us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315960.519952:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4993 00000100:00100000:3.0:1761315960.519970:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4994 00000100:00100000:3.0:1761315960.519974:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049a2c7d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307574848:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00010000:02020000:2.0:1761315960.519995:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315960.519998:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049a2c7d7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307574848:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315960.520009:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4994 00000100:00100000:2.0:1761315960.520029:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307574976:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 122us (189us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315960.520036:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2458 00000100:00100000:3.0:1761315960.520145:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307574784 00000100:00100000:3.0:1761315960.520178:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1833 00000100:00100000:3.0:1761315960.520183:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307574784:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315960.521920:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307574784:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 1730us (2239us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315960.521934:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1833 00010000:00010000:0.0:1761315960.540019:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315961.051587:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315961.051717:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322616512:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315961.051780:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315961.051859:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616512 00000100:00100000:3.0:1761315961.051918:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3115 00000100:00100000:3.0:1761315961.051924:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322616512:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315961.051956:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315961.052563:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322616512:12345-0@lo:41:osp-pre-1-0.0 Request processed in 637us (783us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315961.052572:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3115 00000100:00100000:1.0:1761315961.052741:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322616512:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315961.566488:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761315962.588166:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315963.611492:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315963.611552:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322616576:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315963.611617:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315963.611668:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616576 00000100:00100000:2.0:1761315963.612165:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761315963.612176:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000092fa280b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322616640:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315963.612214:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315963.612241:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322616704:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315963.612247:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315963.612252:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315963.612256:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315963.612261:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4206 00000100:00080000:3.0:1761315963.612264:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315963.612269:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315963.612275:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315963.612294:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315963.612296:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bb399f09 x1846869322616960/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315963.612301:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322616896:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761315963.612331:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315963.612334:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315963.612336:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315963.612340:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4206 00000100:00080000:3.0:1761315963.612341:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315963.612364:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315963.612376:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dad31777 x1846869322617024/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761315963.612395:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612421:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315963.612430:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322617088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612463:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612488:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322616768:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612525:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612540:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322617152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612560:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612577:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761315963.612581:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322616832:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612600:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612613:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322617216:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612633:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612657:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322616960:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612680:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612696:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322617024:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315963.612715:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315963.612748:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617088 00000100:00100000:3.0:1761315963.612785:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616768 00000100:00100000:3.0:1761315963.612799:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617152 00000100:00100000:3.0:1761315963.612807:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616832 00000100:00100000:3.0:1761315963.612820:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4995 00000100:00100000:3.0:1761315963.612825:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322617088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.612861:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616896 00000100:00100000:2.0:1761315963.612887:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616704 00000100:00100000:3.0:1761315963.612895:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008cd9755b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322617088:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (429us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315963.612898:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3116 00000100:00100000:0.0:1761315963.612907:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322616576:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315963.612909:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4995 00000100:00100000:2.0:1761315963.612925:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1834 00000100:00100000:2.0:1761315963.612931:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322616704:12345-0@lo:400:kworker/u8:2.0 00000004:00000024:0.0:1761315963.612935:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315963.612935:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4996 00000100:00100000:3.0:1761315963.612943:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df02dd31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322616768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.612981:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322616576:12345-0@lo:41:osp-pre-0-1.0 Request processed in 74us (1364us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315963.612986:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322616704:12345-0@lo:400:kworker/u8:2.0 Request processed in 53us (710us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315963.612993:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3116 00000100:00100000:2.0:1761315963.612998:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1834 00000100:00100000:0.0:1761315963.613011:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3117 00000100:00100000:0.0:1761315963.613016:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322616896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613044:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322616896:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (650us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315963.613050:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3117 00000100:00100000:2.0:1761315963.613062:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013ebcdeb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322616896:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315963.613074:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322616704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613079:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4997 00000100:00100000:0.0:1761315963.613084:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322617152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613119:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c15e958 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322617152:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (557us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315963.613122:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617216 00000100:00100000:0.0:1761315963.613127:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4997 00000100:00100000:0.0:1761315963.613144:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 4998 00000100:00100000:0.0:1761315963.613147:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322616832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315963.613157:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616960 00000100:00100000:0.0:1761315963.613177:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a3c465f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322616832:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (576us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315963.613177:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617024 00000100:00100000:0.0:1761315963.613185:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4998 00000100:00100000:2.0:1761315963.613198:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1613 00000100:00100000:2.0:1761315963.613203:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322617216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315963.613236:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322617216:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (603us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315963.613242:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1613 00000100:00100000:2.0:1761315963.613252:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2459 00000100:00100000:2.0:1761315963.613254:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322616960:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315963.613364:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df02dd31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322616768:12345-0@lo:400:kworker/u8:2.0 Request processed in 399us (817us total) trans 0 rc 0/0 00010000:02020000:2.0:1761315963.613369:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315963.613374:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 4996 00000100:00100000:2.0:1761315963.613409:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322616960:12345-0@lo:8:kworker/u8:2.0 Request processed in 151us (727us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315963.613414:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322616640 00000100:00100000:2.0:1761315963.613418:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2459 00000100:00100000:2.0:1761315963.613434:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2460 00000100:00100000:2.0:1761315963.613438:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322617024:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613448:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1067 00000100:00100000:0.0:1761315963.613453:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c8247c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322616640:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315963.613471:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315963.613498:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322617088:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613507:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c8247c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322616640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (1292us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315963.613513:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1067 00010000:02020000:2.0:1761315963.613517:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315963.613533:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322616832:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315963.613556:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322616576:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315963.613562:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322617024:12345-0@lo:8:kworker/u8:2.0 Request processed in 121us (844us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315963.613564:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322617216:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315963.613575:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2460 00000100:00100000:3.0:1761315963.613585:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322616768:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315963.613596:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322617152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315963.613619:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000092fa280b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322616640:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761315963.613643:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315963.613647:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315963.613651:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322616960:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315963.613659:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315963.613662:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315963.613664:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322617024:0@lo:8:kworker/u8:2.0 00010000:00010000:2.0:1761315964.637962:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315965.641575:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315965.641662:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315965.642880:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575104 00000100:00100000:1.0:1761315965.646766:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 4999 00000100:00100000:1.0:1761315965.646779:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000092fa280b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307575104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315965.646869:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000092fa280b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307575104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 92us (5302us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315965.646878:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 4999 00000100:00100000:2.0:1761315965.647172:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315965.647201:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315965.647286:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575232 00000100:00100000:0.0:1761315965.647365:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2461 00000100:00100000:0.0:1761315965.647370:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011fac5c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307575232:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315965.647464:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315965.647494:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011fac5c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307575232:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 124us (5834us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315965.647500:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2461 00000100:00100000:0.0:1761315965.647630:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575168 00000100:00100000:0.0:1761315965.647650:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5000 00000100:00100000:0.0:1761315965.647654:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b38f16ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307575168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315965.647677:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b38f16ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307575168:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (476us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315965.647682:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5000 00000100:00100000:3.0:1761315965.648491:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575040 00000100:00100000:3.0:1761315965.648525:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1835 00000100:00100000:3.0:1761315965.648530:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307575040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315965.648565:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307575040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (1395us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315965.648578:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1835 00010000:00010000:1.0:1761315965.660366:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315966.171629:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322617280:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315966.171694:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315966.171796:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617280 00000100:00100000:3.0:1761315966.171850:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3118 00000100:00100000:3.0:1761315966.171856:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322617280:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315966.171889:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315966.171945:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322617280:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (251us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315966.171952:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3118 00000100:00100000:1.0:1761315966.172137:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322617280:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315966.683532:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315967.709770:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761315968.731512:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315968.731519:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761315968.731540:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761315968.731542:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322617344:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315968.731553:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322617408:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315968.731703:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315968.731707:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315968.731769:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617344 00000100:00100000:0.0:1761315968.731862:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1068 00000100:00100000:0.0:1761315968.731868:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322617344:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315968.731897:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315968.731934:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322617344:12345-0@lo:13:osp-pre-1-1.0 Request processed in 65us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315968.731941:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1068 00000100:00080000:2.0:1761315968.731949:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315968.731953:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315968.731956:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315968.731961:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4211 00000100:00080000:2.0:1761315968.731963:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315968.731967:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315968.731984:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dad31777 x1846869322617728/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761315968.731998:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315968.732001:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315968.732003:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315968.732005:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4211 00000100:00080000:2.0:1761315968.732007:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315968.732009:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761315968.732021:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000bb399f09 x1846869322617792/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315968.732041:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322617536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732053:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322617600:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732069:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315968.732078:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322617728:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732101:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315968.732104:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322617344:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761315968.732114:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322617792:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732120:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322617920:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732131:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315968.732136:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315968.732157:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617536 00000100:00100000:2.0:1761315968.732157:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617728 00000100:00100000:1.0:1761315968.732175:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617920 00000100:00100000:2.0:1761315968.732175:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617792 00000100:00100000:1.0:1761315968.732188:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5001 00000100:00100000:2.0:1761315968.732188:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2462 00000100:00100000:1.0:1761315968.732192:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322617536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732192:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322617728:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732222:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322617536:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (153us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732227:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5001 00000100:00100000:1.0:1761315968.732238:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5002 00000100:00100000:1.0:1761315968.732240:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322617920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732260:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322617920:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (124us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732265:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5002 00000100:00100000:1.0:1761315968.732298:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322617920:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761315968.732299:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315968.732307:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322617536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732329:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322617728:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315968.732335:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2462 00000100:00100000:1.0:1761315968.732338:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2463 00000100:00100000:1.0:1761315968.732341:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322617792:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732371:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315968.732385:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322617984:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732403:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315968.732422:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617984 00000100:00100000:2.0:1761315968.732439:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1614 00000100:00100000:2.0:1761315968.732442:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322617984:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761315968.732451:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315968.732471:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322617792:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (339us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315968.732472:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322617984:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (70us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732477:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2463 00000100:00100000:2.0:1761315968.732480:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1614 00000100:00100000:2.0:1761315968.732494:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322617664:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732496:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617600 00000100:00100000:1.0:1761315968.732513:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5003 00000100:00100000:1.0:1761315968.732517:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013ebcdeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322617600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732517:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315968.732532:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761315968.732536:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322617472:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732543:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013ebcdeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322617600:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (172us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732549:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5003 00000100:00100000:2.0:1761315968.732557:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315968.732570:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322617856:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732588:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617472 00000100:00100000:2.0:1761315968.732589:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315968.732606:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1836 00000100:00100000:1.0:1761315968.732610:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322617472:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.732613:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322617600:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732639:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322617472:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (82us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732645:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1836 00000100:00100000:1.0:1761315968.732667:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617856 00000100:00100000:1.0:1761315968.732684:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5004 00000100:00100000:1.0:1761315968.732687:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322617856:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732710:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166de8db pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322617856:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (121us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315968.732715:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5004 00000100:00100000:1.0:1761315968.732737:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322617472:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315968.732745:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322617856:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315968.732777:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315968.732780:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315968.732783:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322617728:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315968.732789:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315968.732791:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315968.732793:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322617792:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315968.732993:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617408 00000100:00100000:3.0:1761315968.733030:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322617664 00000100:00100000:3.0:1761315968.733047:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3119 00000100:00100000:3.0:1761315968.733051:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322617408:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315968.733072:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315968.733108:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322617408:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (1410us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315968.733113:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3119 00000100:00100000:3.0:1761315968.733125:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3120 00000100:00100000:3.0:1761315968.733128:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322617664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315968.733156:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322617664:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (639us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315968.733162:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3120 00000100:00100000:3.0:1761315968.733198:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000300d853f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322617408:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315968.733223:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322617664:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315968.734501:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322617984:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315969.755595:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315970.760264:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315970.760373:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315970.760400:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315970.760436:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315970.760506:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575296 00000100:00100000:3.0:1761315970.760556:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1837 00000100:00100000:3.0:1761315970.760563:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307575296:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315970.760620:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307575296:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 57us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315970.760627:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1837 00000100:00100000:3.0:1761315970.760657:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575424 00000100:00100000:3.0:1761315970.760680:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575360 00000100:00100000:3.0:1761315970.760694:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5005 00000100:00100000:3.0:1761315970.760698:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307575424:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315970.760723:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006142da40 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307575424:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 24us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315970.760729:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5005 00000100:00100000:3.0:1761315970.760738:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5006 00000100:00100000:3.0:1761315970.760742:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307575360:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315970.760757:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007440e179 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307575360:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 15us (322us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315970.760763:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5006 00000100:00100000:0.0:1761315970.761288:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575488 00000100:00100000:0.0:1761315970.761319:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2464 00000100:00100000:0.0:1761315970.761323:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307575488:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761315970.761429:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315970.761472:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307575488:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 148us (1214us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761315970.761479:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2464 00010000:00010000:2.0:1761315970.780012:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315971.293850:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761315971.293873:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322618048:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315971.293941:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315971.294045:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618048 00000100:00100000:2.0:1761315971.294096:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3121 00000100:00100000:2.0:1761315971.294102:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322618048:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315971.294134:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315971.294186:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322618048:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (246us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315971.294193:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3121 00000100:00100000:3.0:1761315971.294221:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322618048:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761315971.803569:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315972.827680:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315973.851604:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618112:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315973.851675:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315973.851729:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322618176:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.851763:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315973.851780:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761315973.851788:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322618240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.851820:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761315973.851829:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315973.851835:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315973.851838:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315973.851843:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322618432:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315973.851845:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4216 00000100:00080000:1.0:1761315973.851850:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315973.851856:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315973.851872:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315973.851878:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000960eacc4 x1846869322618560/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315973.851891:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00080000:1.0:1761315973.851897:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761315973.851899:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322618496:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761315973.851903:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315973.851906:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315973.851911:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4216 00000100:00080000:1.0:1761315973.851914:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315973.851917:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315973.851930:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315973.851933:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f13d776a x1846869322618624/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315973.851981:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322618560:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315973.852011:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315973.852017:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618496 00000100:00100000:3.0:1761315973.852030:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322618624:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315973.852057:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315973.852067:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3122 00000100:00100000:1.0:1761315973.852074:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322618496:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.852090:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618240 00000100:00100000:3.0:1761315973.852139:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618432 00000100:00100000:1.0:1761315973.852142:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322618496:12345-0@lo:400:kworker/u8:0.0 Request processed in 66us (211us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315973.852151:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3122 00000100:00100000:3.0:1761315973.852162:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5007 00000100:00100000:3.0:1761315973.852169:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322618240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.852181:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618560 00000100:00100000:1.0:1761315973.852203:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618624 00000100:00100000:3.0:1761315973.852220:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fed55ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322618240:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315973.852221:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2465 00000100:00100000:1.0:1761315973.852229:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322618560:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315973.852233:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5007 00000100:00100000:3.0:1761315973.852260:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5008 00000100:00100000:3.0:1761315973.852265:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322618432:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.852292:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956dea7b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322618432:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (422us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315973.852301:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5008 00000100:00100000:3.0:1761315973.852335:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618368:0@lo:41:osp-pre-0-1.0 00010000:02020000:1.0:1761315973.852378:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315973.852385:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315973.852405:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322618752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.852425:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322618560:12345-0@lo:8:kworker/u8:0.0 Request processed in 195us (412us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315973.852432:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315973.852436:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2465 00000100:00100000:1.0:1761315973.852452:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2466 00000100:00100000:1.0:1761315973.852455:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322618624:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761315973.852535:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315973.852558:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322618624:12345-0@lo:8:kworker/u8:0.0 Request processed in 102us (501us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315973.852564:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2466 00000100:00080000:1.0:1761315973.852605:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315973.852609:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315973.852612:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322618560:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761315973.852622:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315973.852624:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315973.852626:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f13d776a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322618624:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761315973.852650:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618368 00000100:00100000:1.0:1761315973.852671:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3123 00000100:00100000:1.0:1761315973.852674:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322618368:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761315973.852698:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315973.852842:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618112 00000100:00100000:2.0:1761315973.852873:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1069 00000100:00100000:2.0:1761315973.852877:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322618112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315973.852895:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315973.853306:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618176 00000100:00100000:0.0:1761315973.853334:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1838 00000100:00100000:0.0:1761315973.853337:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322618176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315973.853397:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322618176:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (1635us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315973.853403:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1838 00000100:00100000:0.0:1761315973.853429:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315973.853440:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322618816:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315973.853467:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315973.853490:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618816 00000100:00100000:0.0:1761315973.853517:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1615 00000100:00100000:0.0:1761315973.853522:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322618816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315973.853550:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322618816:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315973.853556:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1615 00000100:00100000:0.0:1761315973.853575:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322618304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315973.853599:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315973.853614:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322618688:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.856227:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322618368:12345-0@lo:41:osp-pre-0-1.0 Request processed in 3543us (3834us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315973.856242:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3123 00000100:00100000:3.0:1761315973.856261:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618368:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315973.856297:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618304 00000100:00100000:3.0:1761315973.856317:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618752 00000100:00100000:1.0:1761315973.856357:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3124 00000100:00100000:1.0:1761315973.856364:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322618304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.856372:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5009 00000100:00100000:3.0:1761315973.856378:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322618752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.856436:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322618304:12345-0@lo:400:kworker/u8:0.0 Request processed in 71us (2837us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315973.856442:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3124 00000100:00100000:3.0:1761315973.856447:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a16ccef pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322618752:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (4015us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315973.856454:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5009 00000100:00100000:1.0:1761315973.856508:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322618496:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315973.856511:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315973.856524:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322618432:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.856539:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322618240:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.856554:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000090e6dce0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322618816:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315973.856556:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d029cd1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322618176:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315973.856792:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322618112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 3912us (5118us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315973.856800:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1069 00000100:00100000:3.0:1761315973.859270:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004ec538ee pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618112:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315973.859308:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000921eafb5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322618752:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315973.859802:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315973.859847:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322618304:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.859922:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618688 00000100:00100000:3.0:1761315973.859975:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5010 00000100:00100000:3.0:1761315973.859980:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322618688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315973.860041:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322618688:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (240us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315973.860048:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5010 00000100:00100000:2.0:1761315973.860156:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009084fbbe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322618688:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761315974.878490:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315975.880302:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315975.880408:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315975.880435:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315975.880516:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315975.880604:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575744 00000100:00100000:3.0:1761315975.880644:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575552 00000100:00100000:1.0:1761315975.880660:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2467 00000100:00100000:1.0:1761315975.880668:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307575744:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:3.0:1761315975.880683:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1839 00000100:00100000:3.0:1761315975.880688:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307575552:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315975.880747:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307575552:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 58us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315975.880756:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1839 00010000:02020000:1.0:1761315975.880761:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315975.880789:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575616 00000100:00100000:1.0:1761315975.880798:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307575744:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 130us (503us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315975.880806:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2467 00000100:00100000:3.0:1761315975.880815:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575680 00000100:00100000:3.0:1761315975.880837:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5011 00000100:00100000:3.0:1761315975.880843:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307575616:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315975.880864:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5012 00000100:00100000:1.0:1761315975.880869:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af5d445a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307575680:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315975.880874:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8da4d9e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307575616:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 31us (467us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315975.880882:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5011 00000100:00100000:1.0:1761315975.880904:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af5d445a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307575680:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315975.880912:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5012 00010000:00010000:0.0:1761315975.899456:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315976.412481:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322618880:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315976.412551:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315976.412659:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618880 00000100:00100000:1.0:1761315976.412722:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3125 00000100:00100000:1.0:1761315976.412729:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322618880:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315976.412760:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315976.416757:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322618880:12345-0@lo:41:osp-pre-1-0.0 Request processed in 4017us (4196us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315976.416775:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3125 00000100:00100000:0.0:1761315976.416893:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322618880:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761315976.923621:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315977.947632:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315978.971460:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315978.971473:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315978.971492:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618944:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315978.971558:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761315978.971787:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315978.971792:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315978.971795:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315978.971801:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4221 00000100:00080000:3.0:1761315978.971804:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315978.971810:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315978.971834:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b8110c0c x1846869322619328/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315978.971862:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315978.971865:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315978.971866:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315978.971870:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4221 00000100:00080000:3.0:1761315978.971871:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315978.971873:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315978.971888:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000036dbb695 x1846869322619392/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315978.972002:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322619072:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972036:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972064:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322619456:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972089:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972107:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761315978.972112:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322619136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972134:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972150:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002eb82f98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322619520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972170:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972203:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619072 00000100:00100000:1.0:1761315978.972253:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619456 00000100:00100000:1.0:1761315978.972270:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619136 00000100:00100000:1.0:1761315978.972281:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619520 00000100:00100000:1.0:1761315978.972293:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5013 00000100:00100000:1.0:1761315978.972298:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f13d776a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322619072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972341:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f13d776a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322619072:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972370:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5013 00000100:00100000:1.0:1761315978.972387:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5014 00000100:00100000:1.0:1761315978.972391:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009084fbbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322619456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972416:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009084fbbe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322619456:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (327us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972422:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5014 00000100:00100000:1.0:1761315978.972432:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5015 00000100:00100000:1.0:1761315978.972435:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322619136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972458:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000921eafb5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322619136:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972463:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5015 00000100:00100000:1.0:1761315978.972474:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5016 00000100:00100000:1.0:1761315978.972477:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322619520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972500:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000090e6dce0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322619520:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (329us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972505:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5016 00000100:00100000:1.0:1761315978.972537:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322619200:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972563:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972578:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322619584:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972601:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972617:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1761315978.972621:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322619008:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972645:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972661:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322619264:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315978.972682:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972698:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322619648:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972719:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972745:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619008 00000100:00100000:1.0:1761315978.972770:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1840 00000100:00100000:1.0:1761315978.972773:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322619008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972803:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322619008:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (158us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972809:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1840 00000100:00100000:1.0:1761315978.972834:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619264 00000100:00100000:1.0:1761315978.972856:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1070 00000100:00100000:1.0:1761315978.972860:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322619264:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761315978.972881:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761315978.972909:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322619264:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (226us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315978.972916:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1070 00000100:00100000:1.0:1761315978.972936:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619584 00000100:00100000:2.0:1761315978.972943:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322619328:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315978.972959:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619648 00000100:00100000:1.0:1761315978.972987:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1616 00000100:00100000:2.0:1761315978.972988:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.972995:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322619584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315978.973012:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322619392:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315978.973036:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322619584:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (433us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315978.973045:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315978.973048:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1616 00000100:00100000:1.0:1761315978.973064:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619328 00000100:00100000:1.0:1761315978.973077:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619392 00000100:00100000:2.0:1761315978.973085:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322618944 00000100:00100000:3.0:1761315978.973155:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1617 00000100:00100000:3.0:1761315978.973159:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322619648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315978.973186:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322619648:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (469us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315978.973191:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1617 00000100:00100000:3.0:1761315978.973200:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2468 00000100:00100000:3.0:1761315978.973203:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322619328:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315978.973291:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315978.973316:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322619328:12345-0@lo:8:kworker/u8:2.0 Request processed in 112us (334us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315978.973321:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2468 00000100:00100000:3.0:1761315978.973328:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2469 00000100:00100000:3.0:1761315978.973331:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322619392:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761315978.973420:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761315978.973438:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322619392:12345-0@lo:8:kworker/u8:2.0 Request processed in 106us (399us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761315978.973443:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2469 00000100:00080000:3.0:1761315978.973475:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315978.973479:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315978.973482:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b8110c0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322619328:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761315978.973488:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761315978.973490:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761315978.973491:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322619392:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761315978.974511:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619200 00000100:00100000:3.0:1761315978.974515:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322619136:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315978.974531:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002eb82f98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322619520:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315978.974546:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322619072:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315978.974549:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3126 00000100:00100000:2.0:1761315978.974556:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322618944:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315978.974561:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322619456:0@lo:400:kworker/u8:2.0 00000004:00000024:2.0:1761315978.974590:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315978.974612:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322619008:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315978.974632:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000765da965 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322619264:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315978.974647:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322619648:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315978.974651:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322618944:12345-0@lo:41:osp-pre-0-1.0 Request processed in 91us (3091us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315978.974661:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3126 00000100:00100000:3.0:1761315978.974664:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322619584:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315978.974681:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3127 00000100:00100000:2.0:1761315978.974687:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322619200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315978.974728:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322619200:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (2164us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315978.974736:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3127 00000100:00100000:3.0:1761315978.974753:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322618944:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315978.974774:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322619200:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761315979.996276:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315981.000277:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315981.000314:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315981.000341:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315981.000408:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315981.000433:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575936 00000100:00100000:0.0:1761315981.000493:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575872 00000100:00100000:0.0:1761315981.000515:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5017 00000100:00100000:0.0:1761315981.000521:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000765da965 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307575936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761315981.000590:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000765da965 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307575936:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 69us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315981.000597:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5017 00000100:00100000:0.0:1761315981.000614:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5018 00000100:00100000:0.0:1761315981.000617:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307575872:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315981.000623:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307575808 00000100:00100000:0.0:1761315981.000641:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087e39f9c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307575872:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 23us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315981.000647:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1841 00000100:00100000:0.0:1761315981.000648:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5018 00000100:00100000:3.0:1761315981.000661:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307575808:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315981.000687:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307575808:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 36us (418us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315981.000693:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1841 00000100:00100000:1.0:1761315981.000789:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576000 00000100:00100000:1.0:1761315981.000823:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2470 00000100:00100000:1.0:1761315981.000827:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307576000:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761315981.000914:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315981.000945:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307576000:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 117us (631us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315981.000951:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2470 00010000:00010000:3.0:1761315981.021111:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315981.532021:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322619712:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761315981.532089:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315981.533562:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619712 00000100:00100000:0.0:1761315981.533618:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3128 00000100:00100000:0.0:1761315981.533624:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322619712:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761315981.533659:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761315981.533711:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322619712:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (1624us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315981.533718:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3128 00000100:00100000:1.0:1761315981.536037:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322619712:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315982.043454:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761315983.076325:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315984.091804:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315984.091811:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315984.091814:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315984.091822:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4226 00000100:00080000:1.0:1761315984.091825:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315984.091830:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315984.091849:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007b71f48d x1846869322620224/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315984.091874:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315984.091877:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315984.091878:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315984.091882:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4226 00000100:00080000:1.0:1761315984.091883:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315984.091886:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315984.091898:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aef6c9f8 x1846869322620288/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315984.091943:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322619968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092005:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315984.092033:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f9477fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322620352:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092055:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315984.092073:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:1.0:1761315984.092079:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322619776:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315984.092102:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315984.092115:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322620032:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092132:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315984.092143:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322620416:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092163:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315984.092228:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619968 00000100:00100000:1.0:1761315984.092265:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620352 00000100:00100000:1.0:1761315984.092281:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620416 00000100:00100000:1.0:1761315984.092298:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5019 00000100:00100000:1.0:1761315984.092303:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322619968:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092385:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322619968:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315984.092393:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5019 00000100:00100000:1.0:1761315984.092409:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5020 00000100:00100000:1.0:1761315984.092412:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322620352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092439:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002716bf0b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322620352:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315984.092457:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5020 00000100:00100000:1.0:1761315984.092468:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5021 00000100:00100000:1.0:1761315984.092471:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322620416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092496:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001dbe4ffe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322620416:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (332us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315984.092501:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5021 00000100:00100000:1.0:1761315984.092549:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322620416:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092561:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322619968:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.092568:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f9477fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322620352:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315984.092649:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007b71f48d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322620224:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315984.092679:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315984.092705:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322620288:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761315984.092725:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315984.092737:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619776 00000100:00100000:2.0:1761315984.092755:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620032 00000100:00100000:0.0:1761315984.092765:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3129 00000100:00100000:0.0:1761315984.092769:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322619776:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761315984.092772:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3130 00000100:00100000:2.0:1761315984.092775:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322620032:12345-0@lo:400:kworker/u8:0.0 00000004:00000024:0.0:1761315984.092791:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315984.092803:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322620032:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (672us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315984.092809:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3130 00000100:00100000:0.0:1761315984.092815:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322619776:12345-0@lo:41:osp-pre-0-1.0 Request processed in 47us (714us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315984.092821:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3129 00000100:00100000:2.0:1761315984.092830:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620224 00000100:00100000:2.0:1761315984.092843:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620288 00000100:00100000:2.0:1761315984.092855:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2471 00000100:00100000:2.0:1761315984.092858:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322620224:12345-0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761315984.092887:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:02020000:2.0:1761315984.092944:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315984.092985:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322620224:12345-0@lo:8:kworker/u8:0.0 Request processed in 125us (305us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315984.092992:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2471 00000100:00100000:2.0:1761315984.093002:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2472 00000100:00100000:2.0:1761315984.093005:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322620288:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761315984.093082:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315984.093104:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322620288:12345-0@lo:8:kworker/u8:0.0 Request processed in 98us (379us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315984.093110:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2472 00000100:00080000:0.0:1761315984.093376:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315984.093381:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315984.093385:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007b71f48d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322620224:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761315984.093393:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761315984.093395:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761315984.093398:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322620288:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093483:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322619904:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093519:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093546:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f69580b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322620160:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093567:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093591:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322620544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093611:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093628:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315984.093634:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322619840:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315984.093656:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093668:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322620096:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093687:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093696:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322620480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093712:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315984.093735:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620544 00000100:00100000:3.0:1761315984.093757:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620480 00000100:00100000:3.0:1761315984.093769:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1618 00000100:00100000:3.0:1761315984.093774:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322620544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093816:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322620544:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (204us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315984.093822:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1618 00000100:00100000:3.0:1761315984.093835:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1619 00000100:00100000:3.0:1761315984.093839:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322620480:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093861:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322620480:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (150us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315984.093866:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1619 00000100:00100000:3.0:1761315984.093909:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322620480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761315984.093918:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322620544:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315984.094054:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619840 00000100:00100000:0.0:1761315984.094086:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1761315984.094089:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322619840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315984.094106:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315984.094136:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322619840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 46us (480us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315984.094141:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1071 00000100:00100000:0.0:1761315984.094156:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620096 00000100:00100000:0.0:1761315984.094187:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5022 00000100:00100000:0.0:1761315984.094191:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322620096:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315984.094215:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a195c8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322620096:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (528us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315984.094220:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5022 00000100:00100000:0.0:1761315984.094240:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322619904 00000100:00100000:0.0:1761315984.094258:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1842 00000100:00100000:0.0:1761315984.094261:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322619904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315984.094284:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322619904:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (766us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315984.094289:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1842 00000100:00100000:0.0:1761315984.094321:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e3856be pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322619840:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315984.094329:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322620096:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761315984.094336:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322619904:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315984.094423:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620160 00000100:00100000:2.0:1761315984.094453:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3131 00000100:00100000:2.0:1761315984.094457:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322620160:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761315984.094491:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322620160:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (923us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315984.094496:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3131 00000100:00100000:2.0:1761315984.094518:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f69580b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322620160:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761315984.095949:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322619776:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315984.095965:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322620032:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761315985.115659:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315986.120543:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315986.120593:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315986.120616:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576256 00000100:00100000:1.0:1761315986.120664:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2473 00000100:00100000:1.0:1761315986.120670:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307576256:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315986.120764:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315986.120830:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307576256:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 159us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315986.120846:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2473 00000100:00100000:0.0:1761315986.121217:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576128 00000100:00100000:0.0:1761315986.121260:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5023 00000100:00100000:0.0:1761315986.121265:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001999cdc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307576128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315986.121309:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001999cdc9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307576128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 43us (716us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315986.121316:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5023 00000100:00100000:3.0:1761315986.121551:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761315986.121633:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761315986.121842:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576064 00000100:00100000:0.0:1761315986.121871:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1843 00000100:00100000:0.0:1761315986.121876:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307576064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761315986.121907:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307576064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315986.121914:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1843 00000100:00100000:1.0:1761315986.122341:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576192 00000100:00100000:1.0:1761315986.122406:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5024 00000100:00100000:1.0:1761315986.122411:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307576192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315986.122444:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004d9b4586 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307576192:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (812us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315986.122451:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5024 00010000:00010000:2.0:1761315986.140229:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315986.651620:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761315986.651643:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322620608:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315986.651706:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315986.651809:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620608 00000100:00100000:2.0:1761315986.651863:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3132 00000100:00100000:2.0:1761315986.651869:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322620608:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761315986.651901:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761315986.651954:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322620608:12345-0@lo:41:osp-pre-1-0.0 Request processed in 85us (249us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315986.651961:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3132 00000100:00100000:3.0:1761315986.652079:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322620608:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315987.163537:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315988.187547:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761315989.211601:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315989.211609:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315989.211613:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315989.211621:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4231 00000100:00080000:1.0:1761315989.211624:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315989.211629:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315989.211647:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004be373e1 x1846869322620992/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761315989.211662:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761315989.211665:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761315989.211666:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761315989.211669:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4231 00000100:00080000:1.0:1761315989.211671:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761315989.211673:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315989.211682:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000adbaff45 x1846869322621056/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315989.211758:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322620736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.211820:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.211849:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f460b6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322621120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.211869:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.211887:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761315989.211891:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f9477fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322620800:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.211913:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.211928:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008843147f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322621184:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.211949:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.211977:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322620992:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212001:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.212015:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322621056:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212033:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315989.212065:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620736 00000100:00100000:1.0:1761315989.212100:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621120 00000100:00100000:1.0:1761315989.212114:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621184 00000100:00100000:1.0:1761315989.212128:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5025 00000100:00100000:1.0:1761315989.212134:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322620736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212212:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015a4137c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322620736:12345-0@lo:400:kworker/u8:2.0 Request processed in 77us (392us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315989.212219:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5025 00000100:00100000:1.0:1761315989.212234:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5026 00000100:00100000:1.0:1761315989.212237:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322621120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212263:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000515354f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322621120:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315989.212268:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5026 00000100:00100000:1.0:1761315989.212277:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5027 00000100:00100000:1.0:1761315989.212280:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322621184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212302:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047e6259e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322621184:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (353us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315989.212307:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5027 00000100:00100000:1.0:1761315989.212329:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620800 00000100:00100000:1.0:1761315989.212369:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3133 00000100:00100000:1.0:1761315989.212372:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322620800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212396:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322620800:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (483us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315989.212401:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3133 00000100:00100000:1.0:1761315989.212419:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620992 00000100:00100000:1.0:1761315989.212432:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621056 00000100:00100000:1.0:1761315989.212441:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2474 00000100:00100000:1.0:1761315989.212444:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322620992:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315989.212522:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315989.212549:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322620992:12345-0@lo:8:kworker/u8:2.0 Request processed in 104us (548us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315989.212554:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2474 00000100:00100000:1.0:1761315989.212562:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2475 00000100:00100000:1.0:1761315989.212564:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322621056:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315989.212667:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315989.212682:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322621056:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (650us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315989.212687:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2475 00000100:00080000:1.0:1761315989.212720:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315989.212723:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315989.212725:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322620992:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315989.212732:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315989.212734:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315989.212736:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000adbaff45 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322621056:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212764:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f9477fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322620800:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212771:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008843147f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322621184:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212777:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aef6c9f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322620736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.212785:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f460b6a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322621120:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.212885:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322620864:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.212916:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.212943:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322621248:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.212964:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.212982:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:2.0:1761315989.212987:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322620672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213010:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.213024:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322620928:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213041:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.213053:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322621312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213071:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.213096:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620928 00000100:00100000:2.0:1761315989.213121:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3134 00000100:00100000:2.0:1761315989.213124:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322620928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213164:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322620928:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (122us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315989.213172:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3134 00000100:00100000:2.0:1761315989.213190:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620672 00000100:00100000:2.0:1761315989.213209:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1844 00000100:00100000:2.0:1761315989.213212:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322620672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213239:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322620672:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (228us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315989.213244:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1844 00000100:00100000:2.0:1761315989.213277:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322620672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213286:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322620928:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213304:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621248 00000100:00100000:2.0:1761315989.213323:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621312 00000100:00100000:2.0:1761315989.213331:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1620 00000100:00100000:2.0:1761315989.213334:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322621248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213401:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322621248:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (436us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315989.213408:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1620 00000100:00100000:2.0:1761315989.213418:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1621 00000100:00100000:2.0:1761315989.213422:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322621312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213445:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322621312:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (374us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315989.213450:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1621 00000100:00100000:2.0:1761315989.213484:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322621312:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315989.213493:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322621248:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315989.213652:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761315989.213898:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761315989.213908:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322621376:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761315989.213936:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315989.213962:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621376 00000100:00100000:3.0:1761315989.213984:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3135 00000100:00100000:3.0:1761315989.213988:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322621376:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761315989.214009:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761315989.214038:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322621376:12345-0@lo:41:osp-pre-0-1.0 Request processed in 49us (102us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315989.214044:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3135 00000100:00100000:3.0:1761315989.214071:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322621376:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761315989.214241:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761315989.214250:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322621440:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761315989.214281:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315989.214362:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621440 00000100:00100000:2.0:1761315989.214391:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1072 00000100:00100000:2.0:1761315989.214395:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322621440:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761315989.214414:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761315989.214444:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322621440:12345-0@lo:13:osp-pre-1-1.0 Request processed in 48us (163us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315989.214450:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1072 00000100:00100000:0.0:1761315989.214518:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac866c31 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322621440:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315989.214622:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322620864 00000100:00100000:1.0:1761315989.214656:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5028 00000100:00100000:1.0:1761315989.214659:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efccd99e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322620864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315989.214826:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efccd99e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322620864:12345-0@lo:400:kworker/u8:2.0 Request processed in 164us (1908us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315989.214832:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5028 00000100:00100000:2.0:1761315989.214892:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322620864:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315990.235533:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315991.248511:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315991.248562:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315991.248591:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315991.248675:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315991.248713:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576512 00000100:00100000:2.0:1761315991.248749:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2476 00000100:00100000:2.0:1761315991.248756:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307576512:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761315991.248852:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761315991.248907:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307576512:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 150us (232us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761315991.248914:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2476 00000100:00100000:3.0:1761315991.249083:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576384 00000100:00100000:3.0:1761315991.249122:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576448 00000100:00100000:3.0:1761315991.249170:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5029 00000100:00100000:3.0:1761315991.249174:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307576384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315991.249212:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac866c31 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307576384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 37us (709us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315991.249218:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5029 00000100:00100000:3.0:1761315991.249230:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5030 00000100:00100000:3.0:1761315991.249234:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307576448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761315991.249256:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e3856be pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307576448:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (666us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315991.249262:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5030 00000100:00100000:1.0:1761315991.249479:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576320 00000100:00100000:1.0:1761315991.249519:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1845 00000100:00100000:1.0:1761315991.249523:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307576320:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761315991.249558:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307576320:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (996us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315991.249565:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1845 00010000:00010000:0.0:1761315991.259553:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315991.771975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322621504:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761315991.772043:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315991.773659:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621504 00000100:00100000:3.0:1761315991.773727:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3136 00000100:00100000:3.0:1761315991.773733:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322621504:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761315991.773769:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761315991.773849:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322621504:12345-0@lo:41:osp-pre-1-0.0 Request processed in 114us (1806us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315991.773856:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3136 00000100:00100000:1.0:1761315991.774002:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322621504:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315992.283643:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315993.307717:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315994.331900:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761315994.331918:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322621568:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315994.331982:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315994.332066:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621568 00000100:00100000:2.0:1761315994.332126:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3137 00000100:00100000:2.0:1761315994.332132:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322621568:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761315994.332166:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315994.332208:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322621568:12345-0@lo:41:osp-pre-0-1.0 Request processed in 76us (227us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315994.332215:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3137 00000100:00080000:3.0:1761315994.332692:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315994.332697:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315994.332700:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315994.332705:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4236 00000100:00080000:3.0:1761315994.332707:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315994.332711:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315994.332730:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a10b3b20 x1846869322621952/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761315994.332743:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761315994.332746:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761315994.332747:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761315994.332750:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4236 00000100:00080000:3.0:1761315994.332752:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761315994.332754:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315994.332764:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002118a78f x1846869322622016/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761315994.333128:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322621824:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.333165:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.333183:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322622208:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.333207:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.333225:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761315994.333231:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322621760:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.333253:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.333265:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322622144:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333438:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621824 00000100:00100000:2.0:1761315994.333467:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621760 00000100:00100000:2.0:1761315994.333486:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5031 00000100:00100000:2.0:1761315994.333489:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b8110c0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322621824:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333526:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b8110c0c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322621824:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (361us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315994.333532:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5031 00000100:00100000:2.0:1761315994.333545:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5032 00000100:00100000:2.0:1761315994.333548:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322621760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333575:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cc929884 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322621760:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (322us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315994.333579:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5032 00000100:00100000:2.0:1761315994.333614:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322621696:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333641:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315994.333658:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322622080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333683:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621696 00000100:00100000:2.0:1761315994.333705:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1846 00000100:00100000:2.0:1761315994.333708:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322621696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333739:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322621696:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (97us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315994.333744:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1846 00000100:00100000:2.0:1761315994.333797:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622208 00000100:00100000:2.0:1761315994.333820:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1622 00000100:00100000:2.0:1761315994.333823:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322622208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.333850:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322622208:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (643us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315994.333855:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1622 00000100:00100000:1.0:1761315994.333973:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322621952:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315994.334002:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315994.334029:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322622016:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315994.334050:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315994.334097:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621952 00000100:00100000:1.0:1761315994.334131:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622016 00000100:00100000:1.0:1761315994.334147:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2477 00000100:00100000:1.0:1761315994.334151:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322621952:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315994.334246:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315994.334279:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322621952:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (276us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315994.334284:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2477 00000100:00100000:1.0:1761315994.334308:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2478 00000100:00100000:1.0:1761315994.334311:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322622016:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761315994.334524:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315994.334550:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322622016:12345-0@lo:8:kworker/u8:2.0 Request processed in 238us (500us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315994.334556:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2478 00000100:00100000:1.0:1761315994.334604:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afcc50ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322621568:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761315994.334634:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315994.334637:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315994.334640:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a10b3b20 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322621952:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761315994.334648:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315994.334650:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315994.334652:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322622016:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761315994.334877:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.334910:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322621824:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.334920:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322622208:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.334938:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fc391ed8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322621760:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.334957:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.335033:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001666cfce pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322621696:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.335043:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761315994.335050:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322621632:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315994.335079:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.335095:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322621888:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.335129:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.335145:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322622272:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.335164:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315994.335189:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622144 00000100:00100000:3.0:1761315994.335218:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622080 00000100:00100000:3.0:1761315994.335233:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5033 00000100:00100000:3.0:1761315994.335237:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322622144:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.335277:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b6ed247 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322622144:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315994.335283:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5033 00000100:00100000:3.0:1761315994.335308:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5034 00000100:00100000:3.0:1761315994.335312:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001d479a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322622080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315994.335642:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621632 00000100:00100000:0.0:1761315994.335672:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1073 00000100:00100000:0.0:1761315994.335676:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322621632:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761315994.335700:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761315994.335743:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322621632:12345-0@lo:13:osp-pre-1-1.0 Request processed in 66us (663us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315994.335749:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1073 00000100:00100000:0.0:1761315994.335782:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322621632:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761315994.335845:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315994.335894:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622272 00000100:00100000:0.0:1761315994.335915:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1623 00000100:00100000:0.0:1761315994.335918:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322622272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315994.335951:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322622272:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (786us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315994.335956:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1623 00000100:00100000:0.0:1761315994.335997:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322622272:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315994.336024:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322621888 00000100:00100000:0.0:1761315994.336054:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3138 00000100:00100000:0.0:1761315994.336058:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322621888:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315994.336085:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322621888:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (958us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315994.336090:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3138 00000100:00100000:0.0:1761315994.336126:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322621888:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315994.336604:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a21a6131 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322622144:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315994.338287:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001d479a5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322622080:12345-0@lo:400:kworker/u8:2.0 Request processed in 2968us (3323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315994.338801:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5034 00000100:00100000:0.0:1761315994.338898:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322622080:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761315995.364490:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761315996.365408:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315996.366944:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576768 00000100:00100000:1.0:1761315996.367012:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2479 00000100:00100000:1.0:1761315996.367018:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307576768:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761315996.367118:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761315996.367197:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307576768:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 177us (1796us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315996.367205:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2479 00000100:00100000:2.0:1761315996.367376:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315996.367442:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576576 00000100:00100000:1.0:1761315996.367477:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1847 00000100:00100000:1.0:1761315996.367482:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307576576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761315996.367513:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307576576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315996.367518:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1847 00000100:00100000:1.0:1761315996.371143:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761315996.371186:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761315996.371282:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576640 00000100:00100000:2.0:1761315996.371335:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576704 00000100:00100000:2.0:1761315996.371392:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5035 00000100:00100000:2.0:1761315996.371398:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307576640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315996.371467:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f460b6a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307576640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (332us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315996.371474:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5035 00000100:00100000:2.0:1761315996.371493:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5036 00000100:00100000:2.0:1761315996.371497:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307576704:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761315996.371520:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008843147f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307576704:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315996.371526:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5036 00010000:00010000:0.0:1761315996.379492:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315996.895410:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322622336:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761315996.895508:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315996.896382:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622336 00000100:00100000:1.0:1761315996.896458:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3139 00000100:00100000:1.0:1761315996.896465:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322622336:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761315996.896503:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761315996.896593:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322622336:12345-0@lo:41:osp-pre-1-0.0 Request processed in 128us (1086us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315996.896600:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3139 00000100:00100000:2.0:1761315996.896804:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000071ef691f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322622336:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761315997.403730:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761315998.427562:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761315999.451694:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322622400:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315999.451793:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315999.453171:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004486994d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322622464:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761315999.453248:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315999.453291:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761315999.453297:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322622656:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315999.453331:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315999.453395:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622464 00000100:00100000:3.0:1761315999.453452:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1074 00000100:00100000:3.0:1761315999.453460:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322622464:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761315999.453497:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761315999.453541:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322622464:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315999.453549:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1074 00000100:00100000:3.0:1761315999.453577:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622656 00000100:00100000:3.0:1761315999.453611:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5037 00000100:00100000:3.0:1761315999.453615:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322622656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315999.453646:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d02f4dd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322622656:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315999.453647:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322622528:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315999.453653:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5037 00000100:00080000:2.0:1761315999.453684:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315999.453696:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761315999.453700:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761315999.453703:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761315999.453704:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322622656:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761315999.453711:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4241 00000100:00080000:2.0:1761315999.453715:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315999.453719:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761315999.453724:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004486994d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322622464:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761315999.453728:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761315999.453733:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322622592:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315999.453747:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322622720:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315999.453755:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315999.453769:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315999.453777:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ac5d130f x1846869322622784/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315999.453784:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622528 00000100:00080000:2.0:1761315999.453799:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761315999.453803:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761315999.453805:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761315999.453809:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4241 00000100:00080000:2.0:1761315999.453811:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761315999.453816:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761315999.453819:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1848 00000100:00100000:3.0:1761315999.453820:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622400 00000100:00100000:1.0:1761315999.453824:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322622528:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315999.453829:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b0c84f13 x1846869322622848/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761315999.453863:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322622528:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (164us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315999.453864:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622720 00000100:00100000:1.0:1761315999.453870:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1848 00000100:00100000:3.0:1761315999.453887:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3140 00000100:00100000:3.0:1761315999.453891:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322622400:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315999.453911:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3141 00000100:00100000:1.0:1761315999.453915:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322622720:12345-0@lo:400:kworker/u8:2.0 00000004:00000024:3.0:1761315999.453916:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761315999.453930:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5dd2733 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322622976:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315999.453942:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322622720:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (172us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315999.453943:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322622400:12345-0@lo:41:osp-pre-0-1.0 Request processed in 52us (2153us total) trans 0 rc 0/0 00000100:00100000:1.0:1761315999.453950:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3141 00000100:00100000:1.0:1761315999.453971:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322622784:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761315999.454075:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761315999.454100:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622592 00000100:00100000:0.0:1761315999.454129:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5038 00000100:00100000:0.0:1761315999.454134:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322622592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315999.454244:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315999.454249:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a21a6131 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322622592:12345-0@lo:400:kworker/u8:2.0 Request processed in 112us (490us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315999.454258:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5038 00000100:00100000:1.0:1761315999.454270:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322622848:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315999.454284:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee2d2bc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322623040:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315999.454297:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761315999.454322:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761315999.454370:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622784 00000100:00100000:0.0:1761315999.454385:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322622400:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761315999.454397:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622848 00000100:00100000:1.0:1761315999.454415:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623040 00000100:00100000:1.0:1761315999.454439:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1624 00000100:00100000:1.0:1761315999.454443:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322623040:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761315999.454479:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322623040:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (159us total) trans 0 rc 0/0 00000100:00100000:2.0:1761315999.454563:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761315999.454586:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761315999.454592:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000906be211 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322622912:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315999.454619:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761315999.454632:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3140 00000100:00100000:2.0:1761315999.454653:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622976 00000100:00100000:3.0:1761315999.454663:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322622912 00000100:00100000:2.0:1761315999.454687:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5039 00000100:00100000:2.0:1761315999.454692:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011ec46ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322622976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761315999.454696:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5040 00000100:00100000:3.0:1761315999.454703:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a9720802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322622912:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315999.454738:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011ec46ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322622976:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1761315999.454754:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a9720802 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322622912:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (135us total) trans 0 rc 0/0 00000100:00100000:0.0:1761315999.454765:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2480 00000100:00100000:1.0:1761315999.454775:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1624 00000100:00100000:0.0:1761315999.454778:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322622784:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761315999.454794:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2481 00000100:00100000:1.0:1761315999.454798:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322622848:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761315999.455145:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315999.455188:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322622784:12345-0@lo:8:kworker/u8:2.0 Request processed in 416us (946us total) trans 0 rc -19/-19 00010000:02020000:1.0:1761315999.455190:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761315999.455200:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2480 00000100:00100000:1.0:1761315999.455228:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322622848:12345-0@lo:8:kworker/u8:2.0 Request processed in 428us (930us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761315999.455235:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2481 00000100:00080000:1.0:1761315999.455273:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315999.455278:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315999.455283:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322622784:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455298:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee2d2bc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322623040:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761315999.455298:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761315999.455303:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761315999.455309:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322622848:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455397:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005ee7850a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322622592:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455421:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052902000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322622528:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455465:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322622720:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455501:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000906be211 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322622912:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761315999.455512:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5dd2733 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322622976:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761315999.455794:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5039 00000100:00100000:3.0:1761315999.455826:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5040 00010000:00010000:0.0:1761316000.476884:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316001.480435:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316001.480487:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316001.480505:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576832 00000100:00100000:1.0:1761316001.480590:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1849 00000100:00100000:0.0:1761316001.480599:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316001.480599:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307576832:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316001.480634:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576960 00000100:00100000:0.0:1761316001.480636:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316001.480668:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577024 00000100:00100000:3.0:1761316001.480679:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307576896 00000100:00100000:1.0:1761316001.480690:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307576832:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 92us (265us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316001.480696:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2482 00000100:00100000:1.0:1761316001.480700:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1849 00000100:00100000:0.0:1761316001.480701:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307577024:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761316001.480711:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5041 00000100:00100000:3.0:1761316001.480717:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307576960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316001.480747:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5042 00000100:00100000:1.0:1761316001.480752:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307576896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316001.480758:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6bb4a39 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307576960:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 40us (270us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316001.480766:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5041 00010000:02020000:0.0:1761316001.480792:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316001.480810:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004486994d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307576896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 57us (211us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316001.480818:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5042 00000100:00100000:0.0:1761316001.480820:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307577024:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 118us (185us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316001.480827:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2482 00010000:00010000:0.0:1761316001.500710:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316002.012339:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322623104:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316002.012492:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316002.012567:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623104 00000100:00100000:1.0:1761316002.012641:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3142 00000100:00100000:1.0:1761316002.012649:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322623104:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316002.012697:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316002.013314:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322623104:12345-0@lo:41:osp-pre-1-0.0 Request processed in 664us (822us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316002.013324:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3142 00000100:00100000:0.0:1761316002.013341:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b53e2c1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322623104:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316002.523598:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316003.547610:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316004.571586:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316004.571594:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316004.571598:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316004.571608:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1761316004.571611:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316004.571619:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316004.571644:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aa5cc910 x1846869322623488/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316004.571661:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316004.571667:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1761316004.571667:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000388257a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322623296:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761316004.571673:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316004.571683:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4247 00000100:00080000:2.0:1761316004.571685:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316004.571687:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316004.571703:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ff653b22 x1846869322623552/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316004.571793:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.571860:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6e8631e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322623680:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.571869:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322623488:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316004.571899:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.571930:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761316004.571939:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002aaef6f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322623360:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.571949:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316004.572031:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623296 00000100:00100000:2.0:1761316004.572108:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3143 00000100:00100000:2.0:1761316004.572114:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322623296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316004.572486:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322623168:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.572657:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff653b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322623552:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316004.572658:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.572689:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322623744:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.572709:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.573007:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.573404:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623680 00000100:00100000:1.0:1761316004.573419:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623360 00000100:00100000:3.0:1761316004.573461:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5043 00000100:00100000:3.0:1761316004.573471:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322623680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.573478:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5044 00000100:00100000:1.0:1761316004.573501:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322623360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.573542:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b53e2c1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322623680:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (1650us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316004.573552:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5043 00000100:00100000:1.0:1761316004.573576:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005ee7850a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322623360:12345-0@lo:400:kworker/u8:0.0 Request processed in 80us (922us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316004.573594:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5044 00000100:00100000:3.0:1761316004.573615:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002aaef6f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322623360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.573635:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6e8631e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322623680:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.573643:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623488 00000100:00100000:1.0:1761316004.573674:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623552 00000100:00100000:1.0:1761316004.573700:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623744 00000100:00100000:1.0:1761316004.573725:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1625 00000100:00100000:1.0:1761316004.573735:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322623744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.573780:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322623744:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (775us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316004.573786:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1625 00000100:00100000:3.0:1761316004.573807:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2483 00000100:00100000:3.0:1761316004.573814:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322623488:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316004.573821:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2484 00000100:00100000:1.0:1761316004.573825:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322623552:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761316004.574123:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316004.574169:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322623488:12345-0@lo:8:kworker/u8:0.0 Request processed in 355us (2227us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316004.574176:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2483 00010000:02020000:1.0:1761316004.574180:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316004.574226:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322623552:12345-0@lo:8:kworker/u8:0.0 Request processed in 399us (1519us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316004.574235:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2484 00000100:00100000:3.0:1761316004.574276:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316004.574298:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3799918 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322623232:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574299:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322623936:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316004.574396:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316004.574405:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316004.574440:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a425d4ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322623616:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574442:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623936 00000100:00100000:1.0:1761316004.574476:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.574487:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1075 00000100:00100000:3.0:1761316004.574494:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322623936:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316004.574503:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322623872:0@lo:41:osp-pre-0-1.0 00002000:00000024:3.0:1761316004.574524:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316004.574579:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322623936:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (185us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316004.574588:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.574592:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1075 00000100:00100000:2.0:1761316004.574602:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322623296:12345-0@lo:400:kworker/u8:0.0 Request processed in 2482us (2806us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316004.574615:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3143 00000100:00100000:0.0:1761316004.574630:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000783e6c5c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322623424:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574639:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322623936:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316004.574674:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.574676:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623232 00000100:00100000:0.0:1761316004.574698:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003d22da5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322623808:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574705:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623616 00000100:00100000:2.0:1761316004.574737:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623424 00000100:00100000:0.0:1761316004.574742:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316004.574742:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5045 00000100:00100000:3.0:1761316004.574750:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322623232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316004.574790:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3144 00000100:00100000:2.0:1761316004.574796:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322623424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574806:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001666cfce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322623232:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (397us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316004.574817:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5045 00000100:00100000:0.0:1761316004.574832:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623168 00000100:00100000:3.0:1761316004.574837:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5046 00000100:00100000:3.0:1761316004.574843:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322623616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316004.574872:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322623424:12345-0@lo:400:kworker/u8:0.0 Request processed in 71us (197us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316004.574883:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3144 00000100:00100000:0.0:1761316004.574887:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1850 00000100:00100000:3.0:1761316004.574893:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fc391ed8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322623616:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316004.574899:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322623168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.574906:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5046 00000100:00100000:2.0:1761316004.574948:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000783e6c5c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322623424:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316004.574960:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322623168:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316004.574974:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1850 00000100:00080000:3.0:1761316004.574974:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316004.574982:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316004.574991:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322623488:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316004.575016:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322623168:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316004.575016:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316004.575025:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316004.575043:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff653b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322623552:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316004.575060:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623808 00000100:00100000:0.0:1761316004.575100:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1626 00000100:00100000:3.0:1761316004.575101:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322623744:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316004.575107:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322623808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316004.575118:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000388257a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322623296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316004.575153:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322623808:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316004.575174:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1626 00000100:00100000:2.0:1761316004.575190:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003d22da5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322623808:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761316004.575255:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316004.575321:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316004.575389:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3799918 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322623232:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316004.575400:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a425d4ca pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322623616:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316004.575491:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322623872 00000100:00100000:0.0:1761316004.575529:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3145 00000100:00100000:0.0:1761316004.575533:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322623872:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316004.575558:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316004.575604:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322623872:12345-0@lo:41:osp-pre-0-1.0 Request processed in 70us (284us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316004.575611:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3145 00000100:00100000:1.0:1761316004.575710:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006cfdec5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322623872:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316005.596465:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316006.606935:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316006.609321:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577088 00000100:00100000:3.0:1761316006.609454:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1851 00000100:00100000:3.0:1761316006.609463:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307577088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316006.609674:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307577088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 212us (2747us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316006.609683:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1851 00000100:00100000:0.0:1761316006.609739:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316006.609805:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577216 00000100:00100000:0.0:1761316006.609817:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316006.609863:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316006.609867:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577152 00000100:00100000:2.0:1761316006.609899:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5047 00000100:00100000:0.0:1761316006.609901:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577280 00000100:00100000:2.0:1761316006.609905:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307577216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316006.609910:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5048 00000100:00100000:1.0:1761316006.609918:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0c9b763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307577152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316006.609955:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003588202b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307577216:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 50us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316006.609962:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0c9b763 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307577152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (144us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316006.609971:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5047 00000100:00100000:1.0:1761316006.609979:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5048 00000100:00100000:0.0:1761316006.610828:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2485 00000100:00100000:0.0:1761316006.610972:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfd0f8a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307577280:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761316006.611079:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316006.611145:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfd0f8a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307577280:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 309us (1283us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316006.611156:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2485 00010000:00010000:0.0:1761316006.619669:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316007.131900:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322624000:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316007.131963:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316007.132273:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624000 00000100:00100000:2.0:1761316007.134522:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3146 00000100:00100000:2.0:1761316007.134534:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322624000:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316007.134578:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316007.134678:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322624000:12345-0@lo:41:osp-pre-1-0.0 Request processed in 145us (2715us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316007.134684:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3146 00000100:00100000:0.0:1761316007.135694:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322624000:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316007.648283:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316008.667644:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316009.691635:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bd68be7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322624064:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316009.691701:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316009.691801:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316009.691864:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624064 00000100:00100000:3.0:1761316009.691920:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1076 00000100:00100000:3.0:1761316009.691926:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322624064:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316009.691958:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316009.692014:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322624064:12345-0@lo:13:osp-pre-1-1.0 Request processed in 87us (313us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316009.692021:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1076 00000100:00100000:1.0:1761316009.692363:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322624128:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316009.692398:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316009.692480:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624128 00000100:00100000:0.0:1761316009.692514:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3147 00000100:00100000:0.0:1761316009.692518:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322624128:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316009.692538:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316009.692581:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322624128:12345-0@lo:41:osp-pre-0-1.0 Request processed in 62us (183us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316009.692587:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3147 00000100:00100000:3.0:1761316009.692668:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322624128:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316009.692841:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bd68be7c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322624064:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316009.692971:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316009.692976:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316009.692979:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316009.692984:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4252 00000100:00080000:3.0:1761316009.692987:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316009.692990:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316009.693017:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000afbb551f x1846869322624448/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316009.693031:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316009.693034:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316009.693035:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316009.693038:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4252 00000100:00080000:3.0:1761316009.693040:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316009.693042:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316009.693050:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c6650b19 x1846869322624512/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316009.693080:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afbb551f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322624448:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316009.693107:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316009.693120:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c6650b19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322624512:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316009.693140:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316009.693161:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059de7950 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322624384:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316009.693181:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316009.693192:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761316009.693197:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000920ff855 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322624320:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316009.693216:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316009.693225:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322624704:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316009.693241:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316009.693503:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322624192:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.693541:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316009.693567:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322624576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.693589:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316009.693606:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761316009.693610:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a0e4173 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322624256:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.693646:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316009.693662:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000059f3b7c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322624640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.693682:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316009.693709:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624448 00000100:00100000:1.0:1761316009.693733:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624512 00000100:00100000:1.0:1761316009.693745:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624704 00000100:00100000:1.0:1761316009.693767:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1627 00000100:00100000:1.0:1761316009.693772:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322624704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.693814:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322624704:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (572us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.693820:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1627 00000100:00100000:1.0:1761316009.693832:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2486 00000100:00100000:1.0:1761316009.693836:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322624448:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316009.693923:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316009.693952:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322624448:12345-0@lo:8:kworker/u8:2.0 Request processed in 115us (845us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316009.693958:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2486 00000100:00100000:1.0:1761316009.693969:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2487 00000100:00100000:1.0:1761316009.693972:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322624512:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316009.694058:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316009.694080:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322624512:12345-0@lo:8:kworker/u8:2.0 Request processed in 108us (940us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316009.694088:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2487 00000100:00100000:1.0:1761316009.694107:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624320 00000100:00100000:1.0:1761316009.694126:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624576 00000100:00100000:1.0:1761316009.694142:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624256 00000100:00100000:1.0:1761316009.694151:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624640 00000100:00100000:1.0:1761316009.694161:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5049 00000100:00100000:1.0:1761316009.694165:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1daabeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322624320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694190:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1daabeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322624320:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (974us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.694196:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5049 00000100:00100000:1.0:1761316009.694207:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5050 00000100:00100000:1.0:1761316009.694210:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322624576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694241:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eb82f98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322624576:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (651us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.694245:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5050 00000100:00100000:1.0:1761316009.694255:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5051 00000100:00100000:1.0:1761316009.694257:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322624256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694278:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084ec238b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322624256:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (632us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.694282:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5051 00000100:00100000:1.0:1761316009.694290:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5052 00000100:00100000:1.0:1761316009.694293:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322624640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694314:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cfdec5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322624640:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (632us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.694318:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5052 00000100:00100000:1.0:1761316009.694383:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624384 00000100:00100000:1.0:1761316009.694405:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3148 00000100:00100000:1.0:1761316009.694409:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322624384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694436:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322624384:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1255us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316009.694441:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3148 00000100:00100000:1.0:1761316009.694473:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000920ff855 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322624320:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694484:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322624704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694492:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059de7950 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322624384:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694522:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a0e4173 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322624256:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694529:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000059f3b7c3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322624640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316009.694537:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322624576:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316009.694698:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316009.694702:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316009.694706:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afbb551f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322624448:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761316009.694714:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316009.694717:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316009.694719:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c6650b19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322624512:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316009.694741:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624192 00000100:00100000:3.0:1761316009.694774:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1852 00000100:00100000:3.0:1761316009.694778:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322624192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316009.694813:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322624192:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (1272us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316009.694818:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1852 00000100:00100000:3.0:1761316009.694849:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322624192:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761316010.720301:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316011.722275:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316011.722457:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316011.722485:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316011.722517:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316011.722563:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577344 00000100:00100000:1.0:1761316011.722605:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1853 00000100:00100000:1.0:1761316011.722611:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307577344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316011.722667:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307577344:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (211us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316011.722673:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1853 00000100:00100000:2.0:1761316011.723001:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577408 00000100:00100000:2.0:1761316011.723044:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577472 00000100:00100000:2.0:1761316011.723066:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5053 00000100:00100000:2.0:1761316011.723071:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9dd87da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307577408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316011.723109:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9dd87da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307577408:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 37us (841us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316011.723115:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5053 00000100:00100000:2.0:1761316011.723132:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5054 00000100:00100000:2.0:1761316011.723137:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5147f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307577472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316011.723161:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5147f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307577472:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (675us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316011.723165:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5054 00000100:00100000:1.0:1761316011.723391:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577536 00000100:00100000:1.0:1761316011.723413:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2488 00000100:00100000:1.0:1761316011.723416:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307577536:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316011.723506:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316011.723535:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307577536:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 117us (1018us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316011.723541:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2488 00010000:00010000:0.0:1761316011.741054:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316012.252577:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1761316012.252600:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322624768:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316012.252681:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316012.252947:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624768 00000100:00100000:2.0:1761316012.253026:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3149 00000100:00100000:2.0:1761316012.253032:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322624768:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316012.253067:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316012.253119:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322624768:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (439us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316012.253125:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3149 00000100:00100000:0.0:1761316012.253238:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322624768:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316012.766386:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316013.789120:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316014.811714:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c6650b19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322624832:0@lo:13:osp-pre-1-1.0 00000100:00080000:2.0:1761316014.811770:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316014.811777:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316014.811780:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316014.811787:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1761316014.811790:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316014.811796:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316014.811821:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316014.811823:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000949f9e44 x1846869322625216/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316014.811839:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316014.811842:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316014.811844:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316014.811847:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4257 00000100:00080000:2.0:1761316014.811849:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316014.811852:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316014.811870:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000cf975079 x1846869322625344/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316014.811876:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322625088:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.811895:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624832 00000100:00100000:1.0:1761316014.811916:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316014.811941:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003d22da5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322624896:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316014.811944:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b576fbf3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322625472:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.811954:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1077 00000100:00100000:0.0:1761316014.811971:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322624832:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316014.811995:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:0.0:1761316014.812007:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316014.812018:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322625152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812055:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316014.812060:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322624832:12345-0@lo:13:osp-pre-1-1.0 Request processed in 97us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316014.812069:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1077 00000100:00100000:2.0:1761316014.812074:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322625536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812097:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316014.812135:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316014.812137:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625152 00000100:00100000:0.0:1761316014.812162:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625088 00000100:00100000:2.0:1761316014.812166:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3150 00000100:00100000:2.0:1761316014.812171:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322625152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812196:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5055 00000100:00100000:0.0:1761316014.812201:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322625088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812212:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322625152:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316014.812220:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3150 00000100:00100000:0.0:1761316014.812238:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f9477fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322625088:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316014.812245:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5055 00000100:00100000:2.0:1761316014.812245:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625536 00000100:00100000:1.0:1761316014.812267:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316014.812271:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000783e6c5c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625024:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812279:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1628 00000100:00100000:2.0:1761316014.812284:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322625536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812303:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316014.812307:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322625088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812326:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322625536:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (228us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316014.812331:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c6650b19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322624832:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316014.812332:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034711e66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322625408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.812340:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1628 00000100:00100000:0.0:1761316014.812387:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316014.812397:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624896 00000100:00100000:2.0:1761316014.812399:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322625152:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812412:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761316014.812415:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322625536:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812421:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322624960:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316014.812434:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1854 00000100:00100000:1.0:1761316014.812439:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322624896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812447:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316014.812467:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625472 00000100:00100000:0.0:1761316014.812472:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625280:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316014.812489:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322624896:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (494us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316014.812496:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625408 00000100:00100000:1.0:1761316014.812497:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1854 00000100:00100000:0.0:1761316014.812503:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316014.812536:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322624960 00000100:00100000:0.0:1761316014.812543:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5056 00000100:00100000:0.0:1761316014.812549:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322625472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316014.812582:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000adbaff45 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322625472:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (318us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316014.812755:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000949f9e44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322625216:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316014.812784:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316014.812800:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf975079 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322625344:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316014.812820:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316014.812854:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625024 00000100:00100000:3.0:1761316014.812875:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625280 00000100:00100000:3.0:1761316014.812884:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3151 00000100:00100000:3.0:1761316014.812888:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322625024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316014.812921:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322625024:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (620us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316014.812927:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3151 00000100:00100000:3.0:1761316014.812938:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3152 00000100:00100000:3.0:1761316014.812940:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322625280:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316014.812967:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5057 00000004:00000024:3.0:1761316014.812968:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316014.812975:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322625408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316014.812999:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322625280:12345-0@lo:41:osp-pre-0-1.0 Request processed in 57us (498us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316014.813006:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3152 00000100:00100000:1.0:1761316014.813020:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0ab56d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322625408:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (634us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316014.813027:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625216 00000100:00100000:1.0:1761316014.813028:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5057 00000100:00100000:3.0:1761316014.813050:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625344 00000100:00100000:3.0:1761316014.813069:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2489 00000100:00100000:1.0:1761316014.813071:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003d22da5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322624896:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316014.813076:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322625216:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316014.813126:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b576fbf3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322625472:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761316014.813164:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316014.813171:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625280:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316014.813182:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000783e6c5c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316014.813189:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034711e66 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322625408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316014.813224:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2490 00000100:00100000:1.0:1761316014.813238:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322625344:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316014.813312:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316014.813365:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322625344:12345-0@lo:8:kworker/u8:2.0 Request processed in 136us (544us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316014.813371:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2490 00000100:00080000:1.0:1761316014.813447:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316014.813451:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316014.813455:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf975079 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322625344:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316014.813903:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5058 00000100:00100000:2.0:1761316014.813910:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd99502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322624960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316014.813953:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd99502 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322624960:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (1508us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316014.813970:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5058 00000100:00100000:2.0:1761316014.813999:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322624960:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316014.814174:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322625216:12345-0@lo:8:kworker/u8:2.0 Request processed in 1098us (1388us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316014.814180:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2489 00000100:00080000:1.0:1761316014.814241:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316014.814246:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316014.814249:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000949f9e44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322625216:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316014.815684:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5056 00010000:00010000:0.0:1761316015.835549:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316016.840565:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316016.840608:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316016.840649:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316016.840689:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316016.840893:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577664 00000100:00100000:3.0:1761316016.840916:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577792 00000100:00100000:2.0:1761316016.840964:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577728 00000100:00100000:3.0:1761316016.840972:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2491 00000100:00100000:1.0:1761316016.840973:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5059 00000100:00100000:3.0:1761316016.840984:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307577792:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:1.0:1761316016.840986:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000381436d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307577664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316016.841001:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5060 00000100:00100000:2.0:1761316016.841007:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009cdb71e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307577728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316016.841072:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000381436d4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307577664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (463us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316016.841073:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009cdb71e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307577728:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 63us (382us total) trans 0 rc 0/0 00010000:02020000:3.0:1761316016.841079:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316016.841090:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5060 00000100:00100000:3.0:1761316016.841120:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307577792:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 138us (563us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316016.841129:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2491 00000100:00100000:2.0:1761316016.841144:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577600 00000100:00100000:2.0:1761316016.841175:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1855 00000100:00100000:2.0:1761316016.841180:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307577600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316016.841212:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307577600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (563us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316016.841222:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1855 00000100:00100000:1.0:1761316016.841866:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5059 00010000:00010000:3.0:1761316016.860893:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316017.371658:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316017.371681:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b93b542 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322625600:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316017.371749:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316017.371778:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625600 00000100:00100000:3.0:1761316017.371828:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3153 00000100:00100000:3.0:1761316017.371834:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322625600:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316017.371875:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316017.371922:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322625600:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (175us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316017.371929:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3153 00000100:00100000:2.0:1761316017.372798:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b93b542 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322625600:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316017.883486:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316018.908586:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316019.931601:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316019.931610:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761316019.931613:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322625664:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316019.931615:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316019.931625:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4262 00000100:00080000:1.0:1761316019.931630:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:0.0:1761316019.931635:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322625920:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316019.931638:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316019.931670:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002e83726e x1846869322626048/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316019.931692:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316019.931696:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316019.931698:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316019.931702:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316019.931705:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4262 00000100:00080000:1.0:1761316019.931708:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316019.931712:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316019.931723:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316019.931736:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000804dae55 x1846869322626112/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316019.931737:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625856:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316019.931790:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316019.931797:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322626240:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.931828:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322625728:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.931829:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322626304:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316019.931840:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316019.931877:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316019.931898:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316019.931942:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625856 00000100:00100000:1.0:1761316019.931957:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322625984:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.931988:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316019.931995:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3154 00000100:00100000:2.0:1761316019.932007:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322625856:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316019.932017:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322626368:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.932035:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625984 00000004:00000024:2.0:1761316019.932039:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316019.932047:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316019.932077:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:1.0:1761316019.932093:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322625792:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.932097:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3155 00000100:00100000:2.0:1761316019.932103:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322625856:12345-0@lo:41:osp-pre-0-1.0 Request processed in 99us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316019.932113:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322625984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932121:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3154 00000100:00100000:1.0:1761316019.932125:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316019.932142:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089c37b13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322626176:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932148:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625792 00000100:00100000:2.0:1761316019.932173:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3156 00000100:00100000:2.0:1761316019.932178:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322625792:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.932195:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322625984:12345-0@lo:400:kworker/u8:0.0 Request processed in 83us (201us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316019.932196:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316019.932207:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3155 00000100:00100000:2.0:1761316019.932251:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322625792:12345-0@lo:400:kworker/u8:0.0 Request processed in 66us (122us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316019.932261:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3156 00000100:00100000:3.0:1761316019.932288:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626304 00000100:00100000:2.0:1761316019.932322:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322625856:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316019.932327:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626368 00000100:00100000:2.0:1761316019.932404:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625664 00000100:00100000:2.0:1761316019.932433:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1856 00000100:00100000:2.0:1761316019.932437:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322625664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932470:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322625664:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (774us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316019.932477:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1856 00000100:00100000:2.0:1761316019.932506:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322625664:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932548:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1629 00000100:00100000:2.0:1761316019.932553:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322626304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932595:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322626304:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (724us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316019.932601:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1629 00000100:00100000:2.0:1761316019.932615:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1630 00000100:00100000:2.0:1761316019.932618:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322626368:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932644:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322626368:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (602us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316019.932649:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1630 00000100:00100000:2.0:1761316019.932672:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322626048:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316019.932695:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316019.932714:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322626112:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316019.932718:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626048 00000100:00100000:2.0:1761316019.932745:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316019.932853:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626112 00000100:00100000:2.0:1761316019.932861:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2492 00000100:00100000:2.0:1761316019.932869:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322626048:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316019.932879:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2493 00000100:00100000:3.0:1761316019.932885:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322626112:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316019.933074:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316019.933109:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322626048:12345-0@lo:8:kworker/u8:0.0 Request processed in 240us (414us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316019.933116:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2492 00000100:00080000:2.0:1761316019.933141:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316019.933144:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316019.933148:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e83726e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322626048:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316019.933175:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322626304:0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761316019.933827:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316019.933905:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322626112:12345-0@lo:8:kworker/u8:0.0 Request processed in 1018us (1161us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316019.933915:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2493 00000100:00080000:3.0:1761316019.933977:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316019.933981:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316019.933985:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322626112:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316019.936725:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626240 00000100:00100000:2.0:1761316019.936740:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625728 00000100:00100000:2.0:1761316019.936814:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626176 00000100:00100000:0.0:1761316019.936886:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322625920 00000100:00100000:2.0:1761316019.937293:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5061 00000100:00100000:2.0:1761316019.937303:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322625728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.937312:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5062 00000100:00100000:3.0:1761316019.937322:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322626240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316019.937478:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5063 00000100:00100000:0.0:1761316019.937484:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322626176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316019.941799:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a10b3b20 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322625728:12345-0@lo:400:kworker/u8:0.0 Request processed in 4489us (9895us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316019.941819:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5061 00000100:00100000:0.0:1761316019.941833:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054511f4f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322626176:12345-0@lo:400:kworker/u8:0.0 Request processed in 4342us (9637us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316019.941852:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5063 00000100:00100000:0.0:1761316019.941899:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5064 00000100:00100000:0.0:1761316019.941907:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322625920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316019.941999:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077854bfe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322625920:12345-0@lo:400:kworker/u8:0.0 Request processed in 94us (10282us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316019.942004:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5064 00000100:00100000:0.0:1761316019.942076:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ec99d931 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322625920:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.942084:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054a8504f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322626240:12345-0@lo:400:kworker/u8:0.0 Request processed in 4754us (10238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316019.942098:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5062 00000100:00100000:0.0:1761316019.942136:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322626240:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316019.942228:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316019.942953:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089c37b13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322626176:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.942966:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e3bfc07b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322626432:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316019.943020:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316019.945507:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322625728:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.945525:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322625984:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.945536:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322626368:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316019.945569:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322625792:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316019.946516:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626432 00000100:00100000:0.0:1761316019.947595:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1078 00000100:00100000:0.0:1761316019.947604:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322626432:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316019.947642:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316019.947723:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322626432:12345-0@lo:13:osp-pre-1-1.0 Request processed in 119us (4703us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316019.947730:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1078 00000100:00100000:1.0:1761316019.949780:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e3bfc07b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322626432:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316020.955538:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316021.965325:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316021.965421:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316021.965466:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316021.965506:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316021.965551:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577920 00000100:00100000:3.0:1761316021.965603:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577984 00000100:00100000:3.0:1761316021.965622:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5065 00000100:00100000:3.0:1761316021.965630:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d70cea3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307577920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316021.965713:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d70cea3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307577920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 83us (401us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316021.965721:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5065 00000100:00100000:3.0:1761316021.965737:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5066 00000100:00100000:3.0:1761316021.965741:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307577984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316021.965761:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307577984:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 19us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316021.965766:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5066 00000100:00100000:2.0:1761316021.966115:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307577856 00000100:00100000:2.0:1761316021.966162:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1857 00000100:00100000:2.0:1761316021.966166:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307577856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316021.966196:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307577856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 30us (731us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316021.966202:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1857 00000100:00100000:1.0:1761316021.966672:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578048 00000100:00100000:1.0:1761316021.966713:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2494 00000100:00100000:1.0:1761316021.966718:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307578048:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316021.966814:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316021.966849:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307578048:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 130us (1428us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316021.966855:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2494 00010000:00010000:0.0:1761316021.979805:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316022.491632:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316022.491670:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003fc163cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322626496:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316022.491776:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316022.491854:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626496 00000100:00100000:0.0:1761316022.491931:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3157 00000100:00100000:0.0:1761316022.491939:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322626496:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316022.491985:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316022.492051:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322626496:12345-0@lo:41:osp-pre-1-0.0 Request processed in 111us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316022.492059:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3157 00000100:00100000:1.0:1761316022.492148:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003fc163cb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322626496:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316023.003557:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316024.027584:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316025.051636:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316025.051666:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006386da6d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322626560:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316025.051669:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000073a4359c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322626624:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316025.051786:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316025.051792:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316025.051794:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316025.051824:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316025.051842:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322626688:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316025.051884:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626560 00000100:00100000:3.0:1761316025.051900:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626624 00000100:00100000:0.0:1761316025.051952:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316025.051976:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1858 00000100:00100000:3.0:1761316025.051985:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322626624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.051991:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322626880:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316025.051998:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316025.051998:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1079 00000100:00080000:1.0:1761316025.052012:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316025.052018:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316025.052018:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322626560:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:1.0:1761316025.052033:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4267 00000100:00100000:0.0:1761316025.052035:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316025.052037:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316025.052043:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316025.052051:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322626624:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316025.052060:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761316025.052063:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1858 00000100:00100000:0.0:1761316025.052076:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322626944:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052077:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007c23855d x1846869322627008/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00002000:00000024:2.0:1761316025.052077:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:1.0:1761316025.052110:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761316025.052112:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316025.052115:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316025.052117:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761316025.052121:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322627008:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316025.052122:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4267 00000100:00080000:1.0:1761316025.052127:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316025.052133:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316025.052152:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a992b434 x1846869322627072/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316025.052160:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626688 00000100:00100000:2.0:1761316025.052160:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322626560:12345-0@lo:13:osp-pre-1-1.0 Request processed in 136us (368us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316025.052178:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316025.052198:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626944 00000100:00100000:2.0:1761316025.052198:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1079 00000100:00100000:3.0:1761316025.052215:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322627072:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052237:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3158 00000100:00100000:1.0:1761316025.052245:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322626816:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052247:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322626688:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316025.052268:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316025.052276:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3159 00000004:00000024:0.0:1761316025.052282:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316025.052283:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316025.052291:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322626944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052310:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322627200:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052324:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322626688:12345-0@lo:41:osp-pre-0-1.0 Request processed in 79us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316025.052324:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627008 00000100:00100000:0.0:1761316025.052332:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3158 00000100:00100000:0.0:1761316025.052375:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626816 00000100:00100000:1.0:1761316025.052377:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316025.052392:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627072 00000100:00100000:0.0:1761316025.052401:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3160 00000100:00100000:0.0:1761316025.052407:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322626816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316025.052417:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322626944:12345-0@lo:400:kworker/u8:2.0 Request processed in 126us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316025.052424:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2495 00000100:00100000:2.0:1761316025.052431:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3159 00000100:00100000:3.0:1761316025.052433:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322627008:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052439:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000073a4359c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322626624:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052444:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322626816:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (164us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316025.052452:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3160 00000100:00100000:1.0:1761316025.052478:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a8f7f27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322626816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052492:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761316025.052499:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322626752:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052506:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626880 00010000:02020000:3.0:1761316025.052531:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316025.052532:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316025.052554:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627200 00000100:00100000:2.0:1761316025.052562:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006386da6d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322626560:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316025.052567:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322627136:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052583:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322626752 00000100:00100000:2.0:1761316025.052597:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316025.052599:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322627008:12345-0@lo:8:kworker/u8:2.0 Request processed in 164us (431us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316025.052609:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316025.052619:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322627264:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052621:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5067 00000100:00100000:3.0:1761316025.052621:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2495 00000100:00100000:0.0:1761316025.052633:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eedc2cf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322626880:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052641:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2496 00000100:00100000:1.0:1761316025.052648:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322627072:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316025.052673:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627136 00000100:00100000:2.0:1761316025.052687:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316025.052689:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eedc2cf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322626880:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (654us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316025.052700:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5067 00000100:00100000:3.0:1761316025.052709:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5068 00000100:00100000:3.0:1761316025.052718:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322627200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052728:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5069 00000100:00100000:0.0:1761316025.052733:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322626752:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:1.0:1761316025.052738:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316025.052782:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5070 00000100:00100000:0.0:1761316025.052792:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a8f7f27b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322626752:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316025.052792:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322627072:12345-0@lo:8:kworker/u8:2.0 Request processed in 143us (539us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316025.052792:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003b24cad0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322627200:12345-0@lo:400:kworker/u8:2.0 Request processed in 70us (410us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316025.052796:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322627136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052818:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2496 00000100:00100000:0.0:1761316025.052822:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5069 00000100:00100000:3.0:1761316025.052832:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5068 00000100:00100000:1.0:1761316025.052839:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627264 00000100:00100000:0.0:1761316025.052869:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322626944:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316025.052883:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1631 00000100:00100000:3.0:1761316025.052902:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322627264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316025.052906:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322626880:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316025.052911:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000073a4359c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322627136:12345-0@lo:400:kworker/u8:2.0 Request processed in 110us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316025.052926:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5070 00000100:00100000:0.0:1761316025.052933:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322626688:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316025.052934:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322626752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052958:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322627136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316025.052972:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322627200:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316025.052978:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322627264:12345-0@lo:400:kworker/u8:2.0 Request processed in 85us (295us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316025.052988:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1631 00000100:00080000:2.0:1761316025.053025:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316025.053032:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316025.053040:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322627072:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761316025.053060:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761316025.053064:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322627264:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316025.053066:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316025.053076:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322627008:0@lo:8:kworker/u8:2.0 00010000:00010000:3.0:1761316026.077003:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316027.080046:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316027.080703:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316027.080799:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316027.080809:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578240 00000100:00100000:1.0:1761316027.080847:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578176 00000100:00100000:0.0:1761316027.080864:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5071 00000100:00100000:0.0:1761316027.080871:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3bfc07b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307578240:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316027.080891:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316027.080905:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5072 00000100:00100000:1.0:1761316027.080911:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307578176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316027.080958:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3bfc07b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307578240:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 86us (256us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316027.080966:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5071 00000100:00100000:0.0:1761316027.081002:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578304 00000100:00100000:0.0:1761316027.081022:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2497 00000100:00100000:0.0:1761316027.081026:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307578304:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761316027.081116:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316027.081169:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307578176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 257us (370us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316027.081177:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5072 00000100:00100000:2.0:1761316027.081250:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578112 00000100:00100000:0.0:1761316027.081316:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307578304:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 288us (441us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316027.081323:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2497 00000100:00100000:2.0:1761316027.081786:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1859 00000100:00100000:2.0:1761316027.081792:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307578112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316027.081839:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307578112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 47us (1802us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316027.081846:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1859 00010000:00010000:0.0:1761316027.099500:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316027.611967:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322627328:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316027.612041:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316027.612774:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627328 00000100:00100000:3.0:1761316027.612823:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3161 00000100:00100000:3.0:1761316027.612830:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322627328:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316027.612865:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316027.612924:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322627328:12345-0@lo:41:osp-pre-1-0.0 Request processed in 93us (884us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316027.612931:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3161 00000100:00100000:2.0:1761316027.613442:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322627328:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316028.124309:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316029.147541:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316030.171524:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322627392:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316030.171591:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.171671:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627392 00000100:00100000:2.0:1761316030.171710:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1080 00000100:00100000:2.0:1761316030.171716:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322627392:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316030.171742:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:1.0:1761316030.171766:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316030.171771:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316030.171774:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316030.171775:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322627392:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (185us total) trans 0 rc 0/0 00000100:00080000:1.0:1761316030.171778:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4272 00000100:00080000:1.0:1761316030.171780:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761316030.171782:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1080 00000100:00080000:1.0:1761316030.171783:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316030.171801:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000069497e7a x1846869322627776/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316030.171801:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322627456:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316030.171814:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316030.171817:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316030.171818:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316030.171820:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4272 00000100:00080000:1.0:1761316030.171822:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316030.171824:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316030.171827:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316030.171833:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c437ee62 x1846869322627840/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316030.171839:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322627712:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.171872:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316030.171879:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322627520:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.171889:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761316030.171893:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322627648:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.171907:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.171917:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627712 00000100:00100000:1.0:1761316030.171925:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322627904:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.171941:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3162 00000100:00100000:1.0:1761316030.171944:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.171944:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322627712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.171957:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761316030.171961:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322627584:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.171971:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322627712:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (100us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.171977:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3162 00000100:00100000:1.0:1761316030.171979:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316030.171989:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322627968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.171989:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627584 00000100:00100000:2.0:1761316030.172005:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3163 00000100:00100000:1.0:1761316030.172006:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.172008:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322627584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.172027:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627520 00000100:00100000:2.0:1761316030.172034:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322627584:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (55us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.172040:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3163 00000100:00100000:1.0:1761316030.172048:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627904 00000100:00100000:2.0:1761316030.172056:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316030.172061:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627968 00000100:00100000:2.0:1761316030.172068:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322628032:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.172072:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627648 00000100:00100000:1.0:1761316030.172082:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5073 00000100:00100000:1.0:1761316030.172085:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf420113 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322627520:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172087:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.172114:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322627712:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172138:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628032 00000100:00100000:2.0:1761316030.172159:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1632 00000100:00100000:2.0:1761316030.172162:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322628032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172193:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322628032:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (105us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.172198:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1632 00000100:00100000:2.0:1761316030.172220:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322627584:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172243:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322627392:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316030.172267:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322628032:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172283:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322627776:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172303:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.172316:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322627840:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172334:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316030.172377:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627776 00000100:00100000:2.0:1761316030.172392:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627840 00000100:00100000:2.0:1761316030.172406:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2498 00000100:00100000:2.0:1761316030.172409:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac5d130f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322627776:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316030.172494:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316030.172523:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5074 00000100:00100000:2.0:1761316030.172524:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac5d130f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322627776:12345-0@lo:8:kworker/u8:2.0 Request processed in 114us (221us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316030.172527:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9e8be3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322627904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172530:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2498 00000100:00100000:2.0:1761316030.172539:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2499 00000100:00100000:2.0:1761316030.172542:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322627840:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316030.172617:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316030.172640:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322627840:12345-0@lo:8:kworker/u8:2.0 Request processed in 97us (306us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316030.172648:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2499 00000100:00080000:2.0:1761316030.172675:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316030.172678:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316030.172681:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5075 00000100:00100000:2.0:1761316030.172681:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322627776:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316030.172686:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000015d77bb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322627968:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316030.172688:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316030.172690:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316030.172692:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322627840:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172714:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322627456 00000100:00100000:2.0:1761316030.172737:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1860 00000100:00100000:2.0:1761316030.172740:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322627456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172773:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322627456:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (946us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.172779:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1860 00000100:00100000:2.0:1761316030.172805:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000c2ca2a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322627456:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316030.172841:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5076 00000100:00100000:2.0:1761316030.172844:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000906be211 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322627648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.173277:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf420113 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322627520:12345-0@lo:400:kworker/u8:2.0 Request processed in 1191us (1369us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316030.173284:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5073 00000100:00100000:1.0:1761316030.173322:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e62725a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322627520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316030.173417:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316030.173425:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322628096:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316030.173447:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316030.173466:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628096 00000100:00100000:1.0:1761316030.173484:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3164 00000100:00100000:1.0:1761316030.173487:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322628096:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316030.173506:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316030.173528:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322628096:12345-0@lo:41:osp-pre-0-1.0 Request processed in 41us (81us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316030.173533:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3164 00000100:00100000:1.0:1761316030.173552:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322628096:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316030.173709:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9e8be3c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322627904:12345-0@lo:400:kworker/u8:2.0 Request processed in 1180us (1764us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316030.173717:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5074 00000100:00100000:3.0:1761316030.173744:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322627904:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316030.175994:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000015d77bb0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322627968:12345-0@lo:400:kworker/u8:2.0 Request processed in 3302us (3981us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.175994:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000906be211 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322627648:12345-0@lo:400:kworker/u8:2.0 Request processed in 3143us (3932us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316030.176007:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5076 00000100:00100000:3.0:1761316030.176055:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322627968:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316030.176103:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5075 00010000:00010000:0.0:1761316030.176181:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316030.176392:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322627648:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316031.195664:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316032.200492:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316032.203215:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316032.203242:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316032.203329:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578368 00000100:00100000:1.0:1761316032.203411:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1861 00000100:00100000:1.0:1761316032.203418:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307578368:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316032.203491:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307578368:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 73us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316032.203498:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1861 00000100:00100000:0.0:1761316032.204333:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578432 00000100:00100000:2.0:1761316032.205192:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5077 00000100:00100000:2.0:1761316032.205201:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059f3b7c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307578432:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316032.205276:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059f3b7c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307578432:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (4791us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316032.205287:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5077 00000100:00100000:3.0:1761316032.207387:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316032.207551:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578496 00000100:00100000:0.0:1761316032.208242:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5078 00000100:00100000:0.0:1761316032.208250:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a0e4173 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307578496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316032.208329:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a0e4173 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307578496:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 80us (5087us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316032.208337:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5078 00000100:00100000:2.0:1761316032.209836:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578560 00000100:00100000:2.0:1761316032.209888:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2500 00000100:00100000:2.0:1761316032.209894:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307578560:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316032.209990:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316032.210050:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307578560:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 155us (2668us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316032.210058:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2500 00010000:00010000:2.0:1761316032.223276:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316032.736562:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316032.737239:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322628160:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316032.737306:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316032.737416:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628160 00000100:00100000:3.0:1761316032.738020:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3165 00000100:00100000:3.0:1761316032.738026:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322628160:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316032.738060:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316032.738167:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322628160:0@lo:41:osp-pre-1-0.0 00000100:00100000:3.0:1761316032.738219:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322628160:12345-0@lo:41:osp-pre-1-0.0 Request processed in 191us (912us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316032.738226:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3165 00010000:00010000:0.0:1761316033.246041:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316034.267890:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316035.291517:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316035.291610:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322628352:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761316035.291616:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316035.291621:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316035.291624:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316035.291629:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4277 00000100:00080000:0.0:1761316035.291631:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316035.291637:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316035.291659:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002f8a797b x1846869322628608/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316035.291672:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316035.291675:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316035.291676:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316035.291679:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4277 00000100:00100000:1.0:1761316035.291680:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316035.291681:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316035.291683:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316035.291692:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1eea702 x1846869322628672/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316035.291712:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322628736:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291732:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d404af5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322628480:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.291733:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.291747:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761316035.291752:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb727457 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322628416:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291758:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316035.291784:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322628864:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291807:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316035.291838:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 4 async RPCs [3->2] 00000100:00100000:0.0:1761316035.291844:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322628288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291871:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316035.291883:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322628224:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316035.291886:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.291904:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322628800:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291920:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.291924:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316035.291932:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322628544:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291952:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.291955:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628352 00000100:00100000:0.0:1761316035.291963:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322628928:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.291979:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.291990:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628736 00000100:00100000:0.0:1761316035.291998:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628864 00000100:00100000:1.0:1761316035.292005:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628480 00000100:00100000:1.0:1761316035.292014:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628800 00000100:00100000:0.0:1761316035.292017:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628928 00000100:00100000:1.0:1761316035.292026:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5079 00000100:00100000:1.0:1761316035.292031:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067f33775 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322628352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.292035:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1633 00000100:00100000:0.0:1761316035.292038:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322628864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316035.292070:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628416 00000100:00100000:0.0:1761316035.292075:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322628864:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.292075:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067f33775 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322628352:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (396us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316035.292084:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1633 00000100:00100000:1.0:1761316035.292084:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5079 00000100:00100000:0.0:1761316035.292101:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1634 00000100:00100000:1.0:1761316035.292101:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5080 00000100:00100000:0.0:1761316035.292106:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322628928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316035.292109:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628224 00000100:00100000:1.0:1761316035.292112:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004a63e2e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322628736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316035.292128:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628544 00000100:00100000:1.0:1761316035.292135:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004a63e2e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322628736:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (403us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.292142:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5080 00000100:00100000:0.0:1761316035.292144:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322628928:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (155us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316035.292148:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3166 00000100:00100000:0.0:1761316035.292157:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1634 00000100:00100000:1.0:1761316035.292159:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5081 00000100:00100000:2.0:1761316035.292159:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062651560 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322628416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292165:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322628480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.292190:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aad6d994 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322628928:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.292212:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d404af5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322628480:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.292220:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062f7f673 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322628864:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316035.292407:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5082 00000100:00100000:0.0:1761316035.292412:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5e706e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322628800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292487:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec99d931 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322628480:12345-0@lo:400:kworker/u8:2.0 Request processed in 322us (728us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.292494:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5081 00000100:00100000:1.0:1761316035.292522:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb727457 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322628416:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292529:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c23855d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322628736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292536:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322628352:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292570:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3167 00000100:00100000:1.0:1761316035.292574:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322628224:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316035.292595:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316035.292622:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322628224:12345-0@lo:41:osp-pre-0-1.0 Request processed in 48us (702us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.292627:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3167 00000100:00100000:1.0:1761316035.292637:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3168 00000100:00100000:1.0:1761316035.292640:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322628544:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292658:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322628544:12345-0@lo:400:kworker/u8:2.0 Request processed in 17us (706us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.292662:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3168 00000100:00100000:1.0:1761316035.292686:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322628224:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316035.292693:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322628544:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292721:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322628608:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292741:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.292752:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322628672:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316035.292768:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.292785:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628608 00000100:00100000:1.0:1761316035.292800:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628672 00000100:00100000:1.0:1761316035.292810:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2501 00000100:00100000:1.0:1761316035.292813:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047308eae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322628608:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316035.292904:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316035.292933:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047308eae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322628608:12345-0@lo:8:kworker/u8:2.0 Request processed in 118us (192us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316035.292938:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2501 00000100:00100000:1.0:1761316035.292948:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2502 00000100:00100000:1.0:1761316035.292951:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322628672:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316035.293027:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316035.293046:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322628672:12345-0@lo:8:kworker/u8:2.0 Request processed in 94us (278us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316035.293051:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2502 00000100:00080000:1.0:1761316035.293070:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316035.293074:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316035.293076:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322628608:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316035.293082:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316035.293084:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316035.293086:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322628672:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316035.293104:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628288 00000100:00100000:1.0:1761316035.293127:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1862 00000100:00100000:1.0:1761316035.293131:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322628288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316035.293162:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322628288:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (1290us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.293167:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1862 00000100:00100000:1.0:1761316035.293200:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322628288:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316035.293292:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316035.293304:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000920ff855 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322628992:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316035.293336:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316035.293392:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322628992 00000100:00100000:1.0:1761316035.293425:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1081 00000100:00100000:1.0:1761316035.293429:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322628992:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316035.293446:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316035.293474:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322628992:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316035.293479:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1081 00000100:00100000:2.0:1761316035.293584:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062651560 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322628416:12345-0@lo:400:kworker/u8:2.0 Request processed in 1427us (1695us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316035.293594:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3166 00000100:00100000:0.0:1761316035.293602:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5e706e7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322628800:12345-0@lo:400:kworker/u8:2.0 Request processed in 1187us (1675us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316035.293611:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5082 00000100:00100000:1.0:1761316035.294186:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a992b434 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322628800:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316035.294676:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000920ff855 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322628992:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316036.315513:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316037.320753:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316037.320796:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316037.320902:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578624 00000100:00100000:2.0:1761316037.320952:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1863 00000100:00100000:2.0:1761316037.320958:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307578624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316037.321126:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307578624:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 167us (379us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316037.321138:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1863 00000100:00100000:3.0:1761316037.321210:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316037.321338:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316037.321391:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578816 00000100:00100000:3.0:1761316037.321419:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2503 00000100:00100000:3.0:1761316037.321423:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307578816:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316037.321513:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316037.321563:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307578816:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 138us (225us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316037.321569:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2503 00000100:00100000:0.0:1761316037.321616:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578688 00000100:00100000:0.0:1761316037.321653:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578752 00000100:00100000:0.0:1761316037.321671:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5083 00000100:00100000:0.0:1761316037.321675:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000976491a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307578688:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316037.321704:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000976491a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307578688:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 29us (908us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316037.321710:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5083 00000100:00100000:0.0:1761316037.321720:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5084 00000100:00100000:0.0:1761316037.321724:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000920ff855 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307578752:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316037.321745:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000920ff855 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307578752:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 21us (537us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316037.321750:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5084 00010000:00010000:0.0:1761316037.347561:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316037.853105:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316037.853126:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322629056:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316037.853193:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316037.853291:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629056 00000100:00100000:2.0:1761316037.853341:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3169 00000100:00100000:2.0:1761316037.853362:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322629056:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316037.853391:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316037.853437:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322629056:12345-0@lo:41:osp-pre-1-0.0 Request processed in 75us (245us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316037.853444:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3169 00000100:00100000:3.0:1761316037.853542:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322629056:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316038.363515:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316039.387504:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316040.411528:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316040.411548:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322629120:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316040.411608:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316040.411750:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629120 00000100:00100000:3.0:1761316040.411831:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1082 00000100:00100000:3.0:1761316040.411838:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322629120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316040.411868:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316040.411921:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322629120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 83us (314us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316040.411933:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1082 00000100:00100000:2.0:1761316040.412024:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322629120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316040.412513:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316040.412524:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322629184:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316040.412559:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316040.412675:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629184 00000100:00100000:2.0:1761316040.412714:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3170 00000100:00100000:2.0:1761316040.412719:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000067f2d4b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322629184:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316040.412745:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316040.412799:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000067f2d4b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322629184:12345-0@lo:41:osp-pre-0-1.0 Request processed in 79us (240us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316040.412806:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3170 00000100:00100000:1.0:1761316040.412925:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000087f8a647 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322629184:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316040.413774:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322629248:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.413827:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316040.414049:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316040.414055:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322629312:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.414082:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316040.414117:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629312 00000100:00100000:1.0:1761316040.414150:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5085 00000100:00100000:1.0:1761316040.414155:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045f0c6ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322629312:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.414196:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045f0c6ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322629312:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (114us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316040.414203:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5085 00000100:00100000:1.0:1761316040.414223:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322629376:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.414249:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316040.414272:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629376 00000100:00100000:1.0:1761316040.414293:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5086 00000100:00100000:1.0:1761316040.414296:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000011000719 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322629376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.414325:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000011000719 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322629376:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (75us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316040.414330:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5086 00000100:00100000:1.0:1761316040.414382:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322629376:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.414404:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322629312:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316040.414598:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316040.414602:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316040.414605:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316040.414610:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4282 00000100:00080000:3.0:1761316040.414612:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316040.414616:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316040.414638:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000fa3b911 x1846869322629504/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316040.414651:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316040.414654:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316040.414712:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316040.414716:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4282 00000100:00080000:3.0:1761316040.414718:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316040.414720:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316040.414733:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000040d7daf8 x1846869322629568/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316040.414777:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006386da6d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629440:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.414822:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316040.414841:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316040.414846:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.414869:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316040.414894:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629760 00000100:00100000:3.0:1761316040.414920:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1635 00000100:00100000:3.0:1761316040.414924:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.414956:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629760:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (87us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316040.414962:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1635 00000100:00100000:3.0:1761316040.414985:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629440 00000100:00100000:3.0:1761316040.415006:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3171 00000100:00100000:3.0:1761316040.415010:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415038:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001b3dd283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629440:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (217us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316040.415044:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3171 00000100:00100000:3.0:1761316040.415063:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322629632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415085:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316040.415101:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761316040.415105:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322629696:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415123:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316040.415146:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629632 00000100:00100000:3.0:1761316040.415168:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629696 00000100:00100000:3.0:1761316040.415182:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5087 00000100:00100000:3.0:1761316040.415185:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322629632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415213:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087dbe404 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322629632:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (128us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316040.415219:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5087 00000100:00100000:3.0:1761316040.415229:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5088 00000100:00100000:3.0:1761316040.415232:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322629696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415256:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045e42374 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322629696:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (132us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316040.415261:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5088 00000100:00100000:3.0:1761316040.415291:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322629696:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415298:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322629632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415319:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000804dae55 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629760:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316040.415325:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006386da6d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629440:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316040.415424:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322629504:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316040.415449:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316040.415469:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322629568:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316040.415484:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316040.415504:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629504 00000100:00100000:1.0:1761316040.415518:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629568 00000100:00100000:1.0:1761316040.415528:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2504 00000100:00100000:1.0:1761316040.415531:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322629504:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316040.415615:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316040.415646:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322629504:12345-0@lo:8:kworker/u8:0.0 Request processed in 114us (197us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316040.415652:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2504 00000100:00100000:2.0:1761316040.415758:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629248 00000100:00100000:2.0:1761316040.415794:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1864 00000100:00100000:2.0:1761316040.415798:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322629248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316040.415834:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322629248:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (2008us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316040.415843:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1864 00000100:00100000:2.0:1761316040.415887:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322629248:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316040.415917:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2505 00000100:00100000:2.0:1761316040.415920:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018109dfd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322629568:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316040.416000:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316040.416021:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018109dfd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322629568:12345-0@lo:8:kworker/u8:0.0 Request processed in 101us (537us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316040.416027:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2505 00000100:00080000:2.0:1761316040.416061:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316040.416064:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316040.416067:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322629504:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761316040.416075:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316040.416077:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316040.416079:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000040d7daf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322629568:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761316040.416195:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316041.439079:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316042.441559:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316042.441752:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316042.441804:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316042.441852:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316042.441887:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578944 00000100:00100000:3.0:1761316042.441929:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579008 00000100:00100000:3.0:1761316042.441948:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5089 00000100:00100000:3.0:1761316042.441953:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040d7daf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307578944:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316042.442026:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040d7daf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307578944:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 72us (274us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316042.442032:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5089 00000100:00100000:3.0:1761316042.442047:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5090 00000100:00100000:3.0:1761316042.442051:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fa3b911 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307579008:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316042.442070:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fa3b911 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307579008:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 18us (230us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316042.442075:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5090 00000100:00100000:2.0:1761316042.446518:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307578880 00000100:00100000:2.0:1761316042.446581:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1865 00000100:00100000:2.0:1761316042.446588:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307578880:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316042.446660:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307578880:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 72us (4856us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316042.446666:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1865 00000100:00100000:0.0:1761316042.447932:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579072 00000100:00100000:0.0:1761316042.447973:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2506 00000100:00100000:0.0:1761316042.447979:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307579072:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761316042.448069:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316042.448125:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307579072:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 145us (6573us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316042.448131:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2506 00010000:00010000:0.0:1761316042.459519:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316042.971551:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629824:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316042.971615:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316042.971693:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629824 00000100:00100000:1.0:1761316042.971744:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3172 00000100:00100000:1.0:1761316042.971749:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629824:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316042.971788:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316042.973463:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322629824:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1708us (1844us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316042.973475:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3172 00000100:00100000:0.0:1761316042.973555:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a7398 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322629824:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316043.484356:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316044.509612:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316045.531684:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316045.531835:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316045.531841:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316045.531844:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316045.531852:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1761316045.531856:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316045.531860:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316045.531882:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8ff6beb x1846869322630208/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316045.531897:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316045.531900:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316045.531901:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316045.531904:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4287 00000100:00080000:1.0:1761316045.531906:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316045.531908:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316045.531919:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a7f128c7 x1846869322630272/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316045.531964:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e2acc1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532028:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316045.532057:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322630400:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532081:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316045.532099:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761316045.532104:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322630080:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532123:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316045.532132:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630464:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532163:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316045.532194:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630016 00000100:00100000:1.0:1761316045.532235:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3173 00000100:00100000:1.0:1761316045.532240:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322630016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532286:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322630016:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.532293:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3173 00000100:00100000:1.0:1761316045.532316:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630400 00000100:00100000:1.0:1761316045.532335:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630080 00000100:00100000:1.0:1761316045.532375:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5091 00000100:00100000:1.0:1761316045.532379:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322630400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532407:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008fce761a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322630400:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.532412:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5091 00000100:00100000:1.0:1761316045.532423:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5092 00000100:00100000:1.0:1761316045.532426:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322630080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532449:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051ddfaba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322630080:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.532457:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5092 00000100:00100000:1.0:1761316045.532474:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630464 00000100:00100000:1.0:1761316045.532493:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1636 00000100:00100000:1.0:1761316045.532496:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322630464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532520:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322630464:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (357us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.532525:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1636 00000100:00100000:1.0:1761316045.532560:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322630080:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532569:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630464:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532576:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e2acc1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316045.532584:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322630400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532673:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322629888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532707:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.532731:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322630144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532765:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.532780:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322630528:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532800:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.532817:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1761316045.532822:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322629952:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532843:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.532854:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322630336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532871:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.532895:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630144 00000100:00100000:3.0:1761316045.532918:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3174 00000100:00100000:3.0:1761316045.532922:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322630144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.532953:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322630144:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (187us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316045.532959:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3174 00000100:00100000:3.0:1761316045.532975:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629952 00000100:00100000:3.0:1761316045.532993:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630336 00000100:00100000:3.0:1761316045.533006:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5093 00000100:00100000:3.0:1761316045.533010:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a7398 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322629952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533037:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a7398 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322629952:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (193us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316045.533042:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5093 00000100:00100000:3.0:1761316045.533053:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5094 00000100:00100000:3.0:1761316045.533056:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322630336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533078:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000804dae55 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322630336:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316045.533083:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5094 00000100:00100000:3.0:1761316045.533101:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630528 00000100:00100000:3.0:1761316045.533117:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1637 00000100:00100000:3.0:1761316045.533120:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322630528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533143:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322630528:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (344us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316045.533149:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1637 00000100:00100000:3.0:1761316045.533177:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322629952:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533185:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322630336:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533192:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322630144:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.533199:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322630528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316045.534519:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322630208:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316045.534554:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322630592:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316045.534586:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316045.534597:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316045.534615:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322630272:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316045.534642:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316045.534649:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630592 00000100:00100000:2.0:1761316045.534679:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630208 00000100:00100000:0.0:1761316045.534698:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3175 00000100:00100000:0.0:1761316045.534705:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322630592:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316045.534705:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630272 00000100:00100000:2.0:1761316045.534721:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2507 00000100:00100000:2.0:1761316045.534726:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322630208:12345-0@lo:8:kworker/u8:0.0 00000004:00000024:0.0:1761316045.534736:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316045.534784:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322630592:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (190us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316045.534791:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3175 00000100:00100000:0.0:1761316045.534833:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322630592:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316045.534854:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2508 00000100:00100000:0.0:1761316045.534857:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322630272:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316045.534860:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316045.534897:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322630208:12345-0@lo:8:kworker/u8:0.0 Request processed in 169us (311us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316045.534904:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2507 00010000:02020000:0.0:1761316045.534931:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:2.0:1761316045.534938:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316045.534943:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316045.534947:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322630208:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316045.534965:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322630272:12345-0@lo:8:kworker/u8:0.0 Request processed in 105us (323us total) trans 0 rc -19/-19 00000100:00080000:2.0:1761316045.534967:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316045.534972:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316045.534975:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2508 00000100:00100000:2.0:1761316045.534978:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322630272:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316045.535994:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316045.536015:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322630656:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316045.536065:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316045.536175:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630656 00000100:00100000:3.0:1761316045.536272:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1083 00000100:00100000:3.0:1761316045.536278:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322630656:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316045.536285:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322629888 00002000:00000024:3.0:1761316045.536310:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316045.536319:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1866 00000100:00100000:1.0:1761316045.536325:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322629888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316045.536694:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cec5f9b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322630656:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316045.537022:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322629888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316045.537568:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322630656:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1285us (1498us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.537569:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322629888:12345-0@lo:400:kworker/u8:0.0 Request processed in 1238us (4856us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316045.537584:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1866 00000100:00100000:3.0:1761316045.537585:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1083 00010000:00010000:0.0:1761316046.555670:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316047.561262:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316047.562600:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316047.563032:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579200 00000100:00100000:2.0:1761316047.563087:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5095 00000100:00100000:2.0:1761316047.563094:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307579200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316047.563174:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cec5f9b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307579200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 79us (1918us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316047.563181:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5095 00000100:00100000:3.0:1761316047.563379:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316047.564232:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579328 00000100:00100000:1.0:1761316047.564258:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2509 00000100:00100000:1.0:1761316047.564262:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307579328:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761316047.565036:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316047.565133:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579264 00000100:00100000:0.0:1761316047.565180:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5096 00000100:00100000:0.0:1761316047.565186:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307579264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316047.565253:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d981f77 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307579264:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 67us (219us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316047.565261:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5096 00000100:00100000:2.0:1761316047.565561:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579136 00000100:00100000:2.0:1761316047.565596:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1867 00000100:00100000:2.0:1761316047.565600:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307579136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316047.565636:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307579136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (2257us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316047.565642:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1867 00010000:02020000:1.0:1761316047.567156:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316047.567217:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307579328:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 2953us (4622us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316047.567229:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2509 00010000:00010000:3.0:1761316047.581628:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316048.091817:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322630720:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316048.091898:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316048.091997:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630720 00000100:00100000:0.0:1761316048.092057:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3176 00000100:00100000:0.0:1761316048.092062:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322630720:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316048.092092:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316048.093232:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322630720:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1164us (1330us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316048.093242:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3176 00000100:00100000:2.0:1761316048.093519:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001e33a0e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322630720:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316048.604456:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316049.629366:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316050.651572:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316050.651579:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316050.651582:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316050.651589:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4293 00000100:00080000:1.0:1761316050.651592:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316050.651598:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316050.651616:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009f52f74a x1846869322631104/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316050.651628:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316050.651630:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316050.651631:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316050.651634:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4293 00000100:00080000:1.0:1761316050.651636:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316050.651638:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316050.651646:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000017a833b2 x1846869322631168/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316050.651764:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.651832:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316050.651868:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322631296:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.651894:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316050.651917:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761316050.651922:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322630976:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.651941:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316050.651955:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322631360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.651977:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316050.652011:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631296 00000100:00100000:3.0:1761316050.652050:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630976 00000100:00100000:3.0:1761316050.652070:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5097 00000100:00100000:3.0:1761316050.652076:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322631296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.652124:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e62725a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322631296:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (229us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316050.652132:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5097 00000100:00100000:3.0:1761316050.652149:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5098 00000100:00100000:3.0:1761316050.652152:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322630976:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.652177:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000c2ca2a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322630976:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (235us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316050.652183:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5098 00000100:00100000:3.0:1761316050.652201:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631360 00000100:00100000:3.0:1761316050.652225:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1761316050.652229:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322631360:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.652258:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322631360:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (280us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316050.652263:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1638 00000100:00100000:3.0:1761316050.652307:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a7f128c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322630976:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.652317:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322631360:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316050.652324:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322631296:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316050.652439:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630912 00000100:00100000:1.0:1761316050.652478:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3177 00000100:00100000:1.0:1761316050.652482:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322630912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316050.652520:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322630912:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (688us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316050.652526:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3177 00000100:00100000:3.0:1761316050.652597:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322630912:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652722:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322630848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652755:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652784:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322631232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652805:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652822:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1761316050.652828:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322630784:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652853:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652867:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322631040:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652886:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652899:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004e2acc1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322631424:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652925:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652949:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322631104:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316050.652972:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.652984:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322631168:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653004:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316050.653044:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630784 00000100:00100000:2.0:1761316050.653069:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1868 00000100:00100000:2.0:1761316050.653073:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322630784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653101:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322630784:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316050.653106:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1868 00000100:00100000:2.0:1761316050.653129:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631424 00000100:00100000:2.0:1761316050.653146:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631104 00000100:00100000:2.0:1761316050.653158:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631168 00000100:00100000:2.0:1761316050.653170:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1639 00000100:00100000:2.0:1761316050.653173:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000075f050ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322631424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653199:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000075f050ac pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322631424:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (274us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316050.653205:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1639 00000100:00100000:2.0:1761316050.653307:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2510 00000100:00100000:2.0:1761316050.653310:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322631104:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316050.653418:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316050.653450:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322631104:12345-0@lo:8:kworker/u8:0.0 Request processed in 139us (478us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316050.653457:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2510 00000100:00100000:2.0:1761316050.653468:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2511 00000100:00100000:2.0:1761316050.653471:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d22da5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322631168:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316050.653553:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316050.653575:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d22da5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322631168:12345-0@lo:8:kworker/u8:0.0 Request processed in 103us (571us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316050.653581:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2511 00000100:00100000:2.0:1761316050.653611:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631040 00000100:00100000:2.0:1761316050.653634:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3178 00000100:00100000:2.0:1761316050.653637:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000783e6c5c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322631040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653664:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000783e6c5c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322631040:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (779us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316050.653671:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3178 00000100:00100000:2.0:1761316050.653688:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322630848 00000100:00100000:2.0:1761316050.653706:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631232 00000100:00100000:2.0:1761316050.653743:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5099 00000100:00100000:2.0:1761316050.653747:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf975079 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322630848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653775:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf975079 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322630848:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (1021us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316050.653781:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5099 00000100:00100000:2.0:1761316050.653791:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5100 00000100:00100000:2.0:1761316050.653795:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034711e66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322631232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653820:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034711e66 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322631232:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (1014us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316050.653825:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5100 00000100:00080000:2.0:1761316050.653852:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316050.653856:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316050.653861:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322631104:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761316050.653870:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316050.653873:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316050.653875:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322631168:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653906:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000de3a181 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322630784:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653915:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8ff6beb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322631040:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653924:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004e2acc1d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322631424:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653933:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322630848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.653942:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322631232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316050.654984:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322631488:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316050.655025:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316050.655199:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316050.655288:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316050.655297:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322631552:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316050.655327:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316050.655385:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631552 00000100:00100000:1.0:1761316050.655415:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3179 00000100:00100000:1.0:1761316050.655419:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322631552:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316050.655443:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316050.655479:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322631552:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316050.655489:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3179 00000100:00100000:1.0:1761316050.655519:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322631552:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316050.658649:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631488 00000100:00100000:3.0:1761316050.658738:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1084 00000100:00100000:3.0:1761316050.658744:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322631488:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316050.658775:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316050.658842:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322631488:12345-0@lo:13:osp-pre-1-1.0 Request processed in 98us (3817us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316050.658849:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1084 00000100:00100000:2.0:1761316050.658931:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322631488:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316051.675776:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316052.681896:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316052.682669:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316052.682755:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579456 00000100:00100000:2.0:1761316052.682822:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5101 00000100:00100000:2.0:1761316052.682828:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307579456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316052.682899:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e33a0e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307579456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (1011us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316052.682906:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5101 00000100:00100000:1.0:1761316052.683337:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579584 00000100:00100000:1.0:1761316052.683386:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2512 00000100:00100000:1.0:1761316052.683391:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307579584:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316052.683479:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316052.683524:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307579584:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 132us (857us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316052.683530:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2512 00000100:00100000:2.0:1761316052.685132:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316052.685216:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579392 00000100:00100000:3.0:1761316052.685259:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1869 00000100:00100000:3.0:1761316052.685265:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307579392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316052.685594:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307579392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 328us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316052.685602:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1869 00000100:00100000:3.0:1761316052.688612:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316052.688704:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579520 00000100:00100000:0.0:1761316052.688807:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5102 00000100:00100000:0.0:1761316052.688814:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307579520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316052.688902:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a37f91e0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307579520:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (297us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316052.688910:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5102 00010000:00010000:2.0:1761316052.699571:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316053.211829:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322631616:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316053.211895:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316053.213977:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631616 00000100:00100000:2.0:1761316053.214032:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3180 00000100:00100000:2.0:1761316053.214038:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095d34b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322631616:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316053.214071:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316053.214134:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095d34b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322631616:12345-0@lo:41:osp-pre-1-0.0 Request processed in 95us (2239us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316053.214140:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3180 00000100:00100000:3.0:1761316053.214241:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322631616:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316053.723495:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316054.747998:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316055.771533:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316055.771542:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322631744:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.771558:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3799918 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322631808:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316055.771563:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316055.771569:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316055.771573:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316055.771579:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4298 00000100:00080000:2.0:1761316055.771583:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316055.771588:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316055.771618:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ac5d130f x1846869322632064/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316055.771632:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316055.771635:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316055.771636:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316055.771639:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4298 00000100:00080000:2.0:1761316055.771640:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316055.771643:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316055.771653:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ee2d2bc1 x1846869322632128/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316055.771719:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322632064:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316055.771815:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000388257a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322631872:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.772687:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.772729:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322632192:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.772760:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.772776:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322632448:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316055.772798:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.772836:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631808 00000100:00100000:1.0:1761316055.772885:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632192 00000100:00100000:1.0:1761316055.772905:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5103 00000100:00100000:1.0:1761316055.772910:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e2acc1d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322631808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.772960:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e2acc1d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322631808:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.772967:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5103 00000100:00100000:1.0:1761316055.772982:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5104 00000100:00100000:1.0:1761316055.772985:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8ff6beb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322632192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773007:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8ff6beb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322632192:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (248us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773013:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5104 00000100:00100000:1.0:1761316055.773030:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632448 00000100:00100000:1.0:1761316055.773052:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1085 00000100:00100000:1.0:1761316055.773056:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322632448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316055.773076:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316055.773094:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.773109:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322632448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 50us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773116:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1085 00000100:00100000:0.0:1761316055.773129:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632000:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773154:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322632192:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773165:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.773175:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005a407033 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322632448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316055.773188:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632384:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773191:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3799918 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322631808:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316055.773226:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.773228:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632000 00000100:00100000:0.0:1761316055.773259:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322631680:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316055.773261:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ee2d2bc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322632128:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773265:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3181 00000100:00100000:1.0:1761316055.773271:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773306:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316055.773306:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.773313:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632000:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (149us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773320:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3181 00000100:00100000:1.0:1761316055.773336:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631680 00000100:00100000:0.0:1761316055.773338:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322631936:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773378:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3182 00000100:00100000:2.0:1761316055.773378:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632064 00000100:00100000:1.0:1761316055.773384:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322631680:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316055.773392:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316055.773401:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632128 00000004:00000024:1.0:1761316055.773406:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316055.773424:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2513 00000100:00100000:0.0:1761316055.773428:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322632320:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316055.773431:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322632064:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773441:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322631680:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (141us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316055.773460:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.773466:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3182 00000100:00100000:1.0:1761316055.773492:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632320 00000100:00100000:0.0:1761316055.773502:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c437ee62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322631680:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316055.773520:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1640 00000100:00100000:1.0:1761316055.773525:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322632320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773536:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2514 00010000:02020000:2.0:1761316055.773536:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316055.773544:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322632128:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773566:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322632320:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (108us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773573:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1640 00000100:00100000:2.0:1761316055.773584:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322632064:12345-0@lo:8:kworker/u8:2.0 Request processed in 153us (359us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316055.773591:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2513 00010000:02020000:0.0:1761316055.773619:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316055.773621:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631936 00000100:00080000:2.0:1761316055.773634:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316055.773639:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316055.773644:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322632064:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773650:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5105 00000100:00100000:1.0:1761316055.773657:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322631936:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316055.773661:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761316055.773663:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322632128:12345-0@lo:8:kworker/u8:2.0 Request processed in 117us (361us total) trans 0 rc -19/-19 00000100:00080000:2.0:1761316055.773665:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316055.773672:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ee2d2bc1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322632128:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773688:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2514 00000100:00100000:1.0:1761316055.773707:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004ec538ee pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322631936:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773714:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5105 00000100:00100000:2.0:1761316055.773718:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316055.773724:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322631936:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773739:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322632320:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773759:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632384 00000100:00100000:2.0:1761316055.773761:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632000:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773787:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631744 00000100:00100000:1.0:1761316055.773790:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1641 00000100:00100000:1.0:1761316055.773795:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.773817:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1870 00000100:00100000:0.0:1761316055.773822:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322631744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.773836:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632384:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (120us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.773843:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1641 00000100:00100000:2.0:1761316055.773845:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632384:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316055.773936:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316055.773951:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6e8631e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322632256:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316055.773971:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316055.775129:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322631872 00000100:00100000:1.0:1761316055.775164:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3183 00000100:00100000:1.0:1761316055.775169:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322631872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316055.775213:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322631872:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (1277us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316055.775219:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3183 00000100:00100000:2.0:1761316055.775331:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632256 00000100:00100000:2.0:1761316055.775386:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5106 00000100:00100000:2.0:1761316055.775390:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322632256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316055.775425:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003f0b33c0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322632256:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (1453us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316055.775431:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5106 00000100:00100000:2.0:1761316055.775475:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000388257a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322631872:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316055.775491:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6e8631e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322632256:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316055.776002:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322631744:12345-0@lo:400:kworker/u8:2.0 Request processed in 2178us (2908us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316055.776011:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1870 00000100:00100000:3.0:1761316055.777246:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322631744:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316056.795762:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316057.800446:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316057.800572:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316057.801196:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316057.801555:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316057.801585:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579840 00000100:00100000:0.0:1761316057.801622:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2515 00000100:00100000:0.0:1761316057.801629:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307579840:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:0.0:1761316057.801731:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316057.801790:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307579840:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 161us (236us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316057.801796:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2515 00000100:00100000:3.0:1761316057.802010:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579648 00000100:00100000:3.0:1761316057.802052:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1871 00000100:00100000:3.0:1761316057.802364:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307579648:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316057.802408:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307579648:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 350us (1969us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316057.802415:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1871 00000100:00100000:1.0:1761316057.802814:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579776 00000100:00100000:1.0:1761316057.802849:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579712 00000100:00100000:1.0:1761316057.802869:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5107 00000100:00100000:1.0:1761316057.802874:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307579776:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316057.802908:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001bb0e258 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307579776:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (2337us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316057.802915:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5107 00000100:00100000:1.0:1761316057.802929:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5108 00000100:00100000:1.0:1761316057.802933:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307579712:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316057.802956:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bfdc3142 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307579712:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (1762us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316057.802962:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5108 00010000:00010000:0.0:1761316057.819636:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316058.331593:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316058.331615:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d87dd10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322632512:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316058.331693:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316058.331789:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632512 00000100:00100000:3.0:1761316058.331838:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3184 00000100:00100000:3.0:1761316058.331844:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f52f74a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322632512:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316058.331875:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316058.331929:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f52f74a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322632512:12345-0@lo:41:osp-pre-1-0.0 Request processed in 84us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316058.331935:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3184 00000100:00100000:0.0:1761316058.331981:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d87dd10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322632512:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316058.845055:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316059.867596:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761316060.891569:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316060.891577:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316060.891580:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316060.891588:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4303 00000100:00080000:0.0:1761316060.891591:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316060.891598:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316060.891617:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d1eea702 x1846869322632896/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316060.891635:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316060.891666:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316060.891668:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316060.891671:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4303 00000100:00080000:0.0:1761316060.891672:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316060.891676:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316060.891688:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002f8a797b x1846869322632960/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316060.891740:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a864682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322632704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.891801:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.891801:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322632640:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.891825:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322633088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.891830:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.891846:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322633024:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.891858:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.891875:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632640 00000100:00100000:0.0:1761316060.891879:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761316060.891883:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322632768:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.891902:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.891909:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633088 00000100:00100000:0.0:1761316060.891912:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb727457 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322633152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.891924:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632768 00000100:00100000:0.0:1761316060.891931:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.891940:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5109 00000100:00100000:3.0:1761316060.891945:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322632640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.891953:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633152 00000100:00100000:0.0:1761316060.891975:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1642 00000100:00100000:0.0:1761316060.891979:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322633152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.891985:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d87dd10 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322632640:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (155us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.891992:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5109 00000100:00100000:3.0:1761316060.892006:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5110 00000100:00100000:0.0:1761316060.892007:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322633152:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (75us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.892010:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322633088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.892012:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1642 00000100:00100000:3.0:1761316060.892031:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf40c82 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322633088:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (174us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.892036:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5110 00000100:00100000:3.0:1761316060.892045:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5111 00000100:00100000:3.0:1761316060.892047:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322632768:12345-0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316060.892060:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316060.892068:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c23855d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322632768:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (165us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.892075:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5111 00000100:00100000:0.0:1761316060.892083:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322632768:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.892092:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb727457 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322633152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.892095:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316060.892100:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322633088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.892106:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322633280:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316060.892128:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316060.892184:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322632896:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316060.892211:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316060.892228:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322632960:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316060.892246:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316060.892271:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633024 00000100:00100000:1.0:1761316060.892369:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632704 00000100:00100000:1.0:1761316060.892398:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3185 00000100:00100000:1.0:1761316060.892406:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322632704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.892443:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322632704:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (643us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316060.892448:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3185 00000100:00100000:2.0:1761316060.892778:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5112 00000100:00100000:2.0:1761316060.892783:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322633024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316060.892812:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005a407033 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322633024:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (717us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316060.892817:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5112 00000100:00100000:1.0:1761316060.892854:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316060.892862:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322633344:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316060.892888:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316060.892909:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633344 00000100:00100000:3.0:1761316060.892915:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322632640:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.892925:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322633024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.892927:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3186 00000100:00100000:1.0:1761316060.892930:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322633344:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316060.892934:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1761316060.892939:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322632576:0@lo:400:kworker/u8:2.0 00000004:00000024:1.0:1761316060.892952:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316060.892963:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316060.892978:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322633344:12345-0@lo:41:osp-pre-0-1.0 Request processed in 46us (91us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.892978:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632832:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.892984:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3186 00000100:00100000:3.0:1761316060.892995:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316060.893006:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d404af5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322633216:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.893007:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000047308eae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322633344:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316060.893022:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632832 00000100:00100000:3.0:1761316060.893024:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316060.893039:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3187 00000100:00100000:1.0:1761316060.893042:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.893044:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633280 00000100:00100000:3.0:1761316060.893064:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1086 00000100:00100000:3.0:1761316060.893068:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322633280:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316060.893069:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322632832:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (75us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316060.893074:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3187 00002000:00000024:3.0:1761316060.893084:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316060.893103:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322632832:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.893108:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322633280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 40us (980us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.893113:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1086 00000100:00100000:1.0:1761316060.893115:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000084899189 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322633280:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316060.893146:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632896 00000100:00100000:1.0:1761316060.893152:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632960 00000100:00100000:3.0:1761316060.893161:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633216 00000100:00100000:1.0:1761316060.893167:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2516 00000100:00100000:1.0:1761316060.893170:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322632896:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316060.893176:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1643 00000100:00100000:3.0:1761316060.893179:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322633216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316060.893207:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322633216:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316060.893212:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1643 00000100:00100000:3.0:1761316060.893222:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2517 00000100:00100000:3.0:1761316060.893224:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322632960:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316060.893273:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316060.893305:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322632896:12345-0@lo:8:kworker/u8:2.0 Request processed in 134us (1094us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316060.893312:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2516 00000100:00100000:1.0:1761316060.893329:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322632576 00000100:00100000:1.0:1761316060.893364:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1872 00000100:00100000:1.0:1761316060.893367:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322632576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.893401:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322632576:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (439us total) trans 0 rc 0/0 00010000:02020000:3.0:1761316060.893401:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316060.893406:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1872 00000100:00100000:3.0:1761316060.893422:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322632960:12345-0@lo:8:kworker/u8:2.0 Request processed in 196us (1176us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316060.893427:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2517 00000100:00080000:1.0:1761316060.893445:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316060.893448:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316060.893451:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322632896:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316060.893458:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316060.893461:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316060.893463:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322632960:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316060.893484:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322632576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316060.893493:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d404af5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322633216:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316060.893739:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a864682 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322632704:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761316061.916221:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316062.925497:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316062.925543:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316062.927169:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580096 00000100:00100000:3.0:1761316062.927328:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2518 00000100:00100000:3.0:1761316062.927336:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307580096:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316062.927477:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316062.927557:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307580096:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 220us (2069us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316062.927565:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2518 00000100:00100000:0.0:1761316062.928170:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316062.928615:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579904 00000100:00100000:1.0:1761316062.928673:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1873 00000100:00100000:1.0:1761316062.928679:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307579904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316062.928735:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307579904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 56us (3193us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316062.928742:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1873 00000100:00100000:0.0:1761316062.929049:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316062.930157:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580032 00000100:00100000:3.0:1761316062.930206:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307579968 00000100:00100000:1.0:1761316062.931051:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5113 00000100:00100000:1.0:1761316062.931063:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307580032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316062.931148:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a864682 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307580032:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 86us (2979us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316062.931156:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5113 00000100:00100000:1.0:1761316062.931177:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5114 00000100:00100000:1.0:1761316062.931181:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307579968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316062.931206:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d404af5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307579968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (2159us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316062.931212:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5114 00010000:00010000:3.0:1761316062.940259:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316063.451697:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316063.451820:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322633408:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316063.451886:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316063.451982:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633408 00000100:00100000:2.0:1761316063.452035:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3188 00000100:00100000:2.0:1761316063.452041:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322633408:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316063.452072:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316063.452120:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322633408:12345-0@lo:41:osp-pre-1-0.0 Request processed in 78us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316063.452128:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3188 00000100:00100000:0.0:1761316063.452204:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322633408:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316063.963510:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316064.989085:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316066.012067:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316066.012074:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316066.012078:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316066.012086:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4308 00000100:00080000:2.0:1761316066.012089:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316066.012095:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316066.012116:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001f2449da x1846869322633792/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316066.012129:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316066.012132:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316066.012133:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316066.012136:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4308 00000100:00080000:2.0:1761316066.012138:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316066.012140:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316066.012151:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000aa5cc910 x1846869322633856/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316066.012198:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322633600:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012266:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012292:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322633984:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012315:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012333:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761316066.012338:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322633664:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012389:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012404:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634048:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012426:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012457:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633984 00000100:00100000:2.0:1761316066.012492:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633664 00000100:00100000:2.0:1761316066.012512:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5115 00000100:00100000:2.0:1761316066.012518:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322633984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012525:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322633536:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012554:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012560:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce504304 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322633984:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316066.012568:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322633920:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012570:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5115 00000100:00100000:2.0:1761316066.012586:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633536 00000100:00100000:1.0:1761316066.012587:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012597:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633920 00000100:00100000:1.0:1761316066.012599:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1761316066.012615:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322633472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012620:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5116 00000100:00100000:2.0:1761316066.012624:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322633664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012642:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012647:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e83726e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322633664:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (258us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316066.012653:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5116 00000100:00100000:1.0:1761316066.012654:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322633728:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012674:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012677:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5117 00000100:00100000:2.0:1761316066.012680:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000047308eae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322633536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012687:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322634112:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012705:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000047308eae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322633536:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316066.012707:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316066.012710:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5117 00000100:00100000:2.0:1761316066.012720:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5118 00000100:00100000:2.0:1761316066.012723:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018109dfd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322633920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012733:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b0c84f13 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322633536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012744:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018109dfd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322633920:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (157us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316066.012749:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5118 00000100:00100000:1.0:1761316066.012752:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322633920:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012769:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322633792:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316066.012777:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634048 00000100:00100000:1.0:1761316066.012790:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316066.012804:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322633856:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316066.012824:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761316066.012992:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316066.013008:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633472 00000100:00100000:3.0:1761316066.013033:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1874 00000100:00100000:3.0:1761316066.013037:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322633472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.013074:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322633472:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316066.013082:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1874 00000100:00100000:0.0:1761316066.013439:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633600 00000100:00100000:0.0:1761316066.013461:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633728 00000100:00100000:0.0:1761316066.013473:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3189 00000100:00100000:0.0:1761316066.013476:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322633600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316066.013504:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322633600:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (1239us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316066.013508:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3189 00000100:00100000:0.0:1761316066.013517:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3190 00000100:00100000:0.0:1761316066.013520:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322633728:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316066.013536:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322633728:12345-0@lo:400:kworker/u8:2.0 Request processed in 16us (863us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316066.013540:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3190 00000100:00100000:0.0:1761316066.013555:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316066.013562:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322634240:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316066.013583:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316066.013612:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634240 00000100:00100000:0.0:1761316066.013631:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1087 00000100:00100000:0.0:1761316066.013634:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322634240:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316066.013651:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316066.013687:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322634240:12345-0@lo:13:osp-pre-1-1.0 Request processed in 52us (104us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316066.013693:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1087 00000100:00100000:0.0:1761316066.013719:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322634240:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316066.020136:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634112 00000100:00100000:2.0:1761316066.020193:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633792 00000100:00100000:2.0:1761316066.020202:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322633856 00000100:00100000:1.0:1761316066.020208:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1644 00000100:00100000:1.0:1761316066.020215:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322634048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316066.020218:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1645 00000100:00100000:2.0:1761316066.020221:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322634112:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316066.020278:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322634048:12345-0@lo:400:kworker/u8:2.0 Request processed in 62us (7851us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316066.020278:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322634112:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (7572us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316066.020285:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1644 00000100:00100000:2.0:1761316066.020287:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1645 00000100:00100000:2.0:1761316066.020300:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2519 00000100:00100000:1.0:1761316066.020303:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2520 00000100:00100000:2.0:1761316066.020304:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322633792:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316066.020306:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069497e7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322633856:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316066.020491:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:1.0:1761316066.020492:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316066.020522:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069497e7a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322633856:12345-0@lo:8:kworker/u8:2.0 Request processed in 215us (7698us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316066.020528:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2520 00000100:00100000:2.0:1761316066.021318:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322633792:12345-0@lo:8:kworker/u8:2.0 Request processed in 1011us (8525us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316066.021325:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2519 00000100:00100000:3.0:1761316066.021488:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322634176:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316066.021530:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316066.021564:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a3db8e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322633472:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.021577:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f0d14a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322633728:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.021588:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322634112:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.021641:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634176 00000100:00100000:3.0:1761316066.021671:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3191 00000100:00100000:3.0:1761316066.021675:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322634176:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316066.021700:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316066.021735:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322634176:12345-0@lo:41:osp-pre-0-1.0 Request processed in 59us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316066.021741:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3191 00000100:00100000:3.0:1761316066.021773:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002f8a797b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322634176:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761316066.021949:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316066.021953:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316066.021957:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001f2449da pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322633792:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316066.021968:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316066.021970:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316066.021972:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa5cc910 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322633856:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316066.022765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ea70040 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322633664:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.022778:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.022787:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ac5d130f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322633600:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316066.022797:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322633984:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316067.035517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316068.042524:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316068.042569:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316068.042602:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580160 00000100:00100000:1.0:1761316068.042615:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316068.042667:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316068.042680:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1875 00000100:00100000:2.0:1761316068.042687:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307580160:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316068.042694:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580352 00000100:00100000:1.0:1761316068.042715:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2521 00000100:00100000:1.0:1761316068.042719:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307580352:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:2.0:1761316068.042750:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307580160:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316068.042757:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1875 00010000:02020000:1.0:1761316068.042822:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316068.042825:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580288 00000100:00100000:1.0:1761316068.042855:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307580352:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 134us (187us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316068.042862:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2521 00000100:00100000:0.0:1761316068.042872:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580224 00000100:00100000:3.0:1761316068.042878:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5119 00000100:00100000:3.0:1761316068.042885:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a7f128c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307580288:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316068.042904:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5120 00000100:00100000:0.0:1761316068.042909:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c437ee62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307580224:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316068.042926:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a7f128c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307580288:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 40us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316068.042934:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5119 00000100:00100000:0.0:1761316068.042944:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c437ee62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307580224:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (327us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316068.042952:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5120 00010000:00010000:1.0:1761316068.073018:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316068.572133:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322634304:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316068.572200:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316068.572305:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634304 00000100:00100000:3.0:1761316068.572380:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3192 00000100:00100000:3.0:1761316068.572386:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322634304:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316068.572418:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316068.572475:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322634304:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (275us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316068.572482:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3192 00000100:00100000:1.0:1761316068.572527:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000053fa0bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322634304:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316069.085306:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316070.107503:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316071.131675:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316071.131681:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316071.131685:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316071.131693:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4313 00000100:00080000:2.0:1761316071.131696:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316071.131702:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316071.131721:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004654d769 x1846869322634688/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316071.131731:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316071.131734:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316071.131735:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316071.131739:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4313 00000100:00080000:2.0:1761316071.131740:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316071.131743:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316071.131752:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004870ba5e x1846869322634752/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316071.131792:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f69580b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634496:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.131854:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316071.131877:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322634880:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.131900:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316071.131919:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761316071.131924:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322634560:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.131944:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316071.131959:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634944:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.131978:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316071.132008:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634880 00000100:00100000:2.0:1761316071.132042:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634560 00000100:00100000:2.0:1761316071.132059:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5121 00000100:00100000:2.0:1761316071.132064:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322634880:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.132106:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000052902000 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322634880:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (205us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.132112:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5121 00000100:00100000:2.0:1761316071.132126:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5122 00000100:00100000:2.0:1761316071.132129:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322634560:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.132149:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a6de75ec pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322634560:12345-0@lo:400:kworker/u8:2.0 Request processed in 19us (205us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.132157:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5122 00000100:00100000:2.0:1761316071.132190:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322634560:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.132198:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322634880:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132283:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322634432:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132311:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132335:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322634816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132378:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132392:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1761316071.132397:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322634368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132419:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132429:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322634624:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132444:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132452:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5a9c309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322635008:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132466:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132483:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322634688:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132501:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132512:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322634752:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132527:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316071.132549:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634944 00000100:00100000:1.0:1761316071.132582:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635008 00000100:00100000:1.0:1761316071.132597:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634688 00000100:00100000:1.0:1761316071.132605:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634752 00000100:00100000:1.0:1761316071.132614:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1646 00000100:00100000:1.0:1761316071.132617:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322634944:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132647:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322634944:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (668us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316071.132653:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1646 00000100:00100000:1.0:1761316071.132664:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1647 00000100:00100000:1.0:1761316071.132667:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f2449da pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322635008:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.132691:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f2449da pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322635008:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (225us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316071.132696:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1647 00000100:00100000:1.0:1761316071.132706:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2522 00000100:00100000:1.0:1761316071.132710:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322634688:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316071.132791:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316071.132824:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322634688:12345-0@lo:8:kworker/u8:2.0 Request processed in 113us (322us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316071.132830:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2522 00000100:00100000:1.0:1761316071.132840:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2523 00000100:00100000:1.0:1761316071.132843:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322634752:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316071.132914:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316071.132936:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322634752:12345-0@lo:8:kworker/u8:2.0 Request processed in 92us (408us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316071.132941:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2523 00000100:00080000:1.0:1761316071.132969:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316071.132973:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316071.132976:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322634688:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316071.132983:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316071.132985:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316071.132987:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322634752:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316071.133014:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5a9c309 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322635008:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316071.133097:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634496 00000100:00100000:3.0:1761316071.133135:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634624 00000100:00100000:3.0:1761316071.133153:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3193 00000100:00100000:3.0:1761316071.133156:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322634496:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316071.133188:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062800435 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322634496:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (1335us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316071.133194:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3193 00000100:00100000:3.0:1761316071.133205:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3194 00000100:00100000:3.0:1761316071.133208:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322634624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316071.133237:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322634624:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (792us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316071.133242:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3194 00000100:00100000:3.0:1761316071.133270:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322634624:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.133378:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634368 00000100:00100000:2.0:1761316071.133406:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1876 00000100:00100000:2.0:1761316071.133409:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322634368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.133438:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322634368:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (1018us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.133443:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1876 00000100:00100000:2.0:1761316071.133470:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634432 00000100:00100000:2.0:1761316071.133486:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322634816 00000100:00100000:2.0:1761316071.133497:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5123 00000100:00100000:2.0:1761316071.133500:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ac5d130f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322634432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.133522:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ac5d130f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322634432:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (1211us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.133526:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5123 00000100:00100000:2.0:1761316071.133534:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5124 00000100:00100000:2.0:1761316071.133536:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322634816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.133553:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ea70040 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322634816:12345-0@lo:400:kworker/u8:2.0 Request processed in 14us (1174us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.133556:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5124 00000100:00100000:2.0:1761316071.133597:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634944:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316071.133606:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f69580b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322634496:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.133677:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e4a4d687 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322634368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.133687:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322634432:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.133696:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322634816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316071.134016:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322635072:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316071.134047:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316071.134133:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316071.134141:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322635136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316071.134169:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316071.134212:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635072 00000100:00100000:3.0:1761316071.134234:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3195 00000100:00100000:3.0:1761316071.134237:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322635072:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316071.134261:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316071.134367:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635136 00000100:00100000:2.0:1761316071.134393:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1088 00000100:00100000:2.0:1761316071.134396:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322635136:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316071.134415:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316071.134447:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322635136:12345-0@lo:13:osp-pre-1-1.0 Request processed in 49us (278us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316071.134453:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1088 00000100:00100000:2.0:1761316071.134486:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322635136:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316071.134917:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322635072:12345-0@lo:41:osp-pre-0-1.0 Request processed in 678us (870us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316071.134924:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3195 00000100:00100000:1.0:1761316071.135394:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322635072:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761316071.136267:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316072.156603:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316073.163780:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316073.163843:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316073.165984:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580480 00000100:00100000:0.0:1761316073.166055:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5125 00000100:00100000:0.0:1761316073.166062:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307580480:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316073.166140:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000071ef691f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307580480:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 78us (2367us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316073.166147:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5125 00000100:00100000:2.0:1761316073.166529:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316073.166573:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580416 00000100:00100000:1.0:1761316073.166615:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1877 00000100:00100000:1.0:1761316073.166620:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307580416:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316073.166652:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307580416:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 32us (2811us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316073.166658:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1877 00000100:00100000:3.0:1761316073.166713:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580544 00000100:00100000:3.0:1761316073.166741:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5126 00000100:00100000:3.0:1761316073.166746:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307580544:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316073.166777:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f69580b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307580544:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 31us (250us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316073.166784:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5126 00000100:00100000:1.0:1761316073.166890:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316073.167419:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580608 00000100:00100000:2.0:1761316073.167451:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2524 00000100:00100000:2.0:1761316073.167455:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307580608:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761316073.167541:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316073.167594:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307580608:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 138us (705us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316073.167609:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2524 00010000:00010000:0.0:1761316073.180716:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316073.692142:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316073.692165:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322635200:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316073.692230:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316073.692753:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635200 00000100:00100000:3.0:1761316073.692804:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3196 00000100:00100000:3.0:1761316073.692810:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322635200:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316073.692843:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316073.692897:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322635200:12345-0@lo:41:osp-pre-1-0.0 Request processed in 86us (668us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316073.692904:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3196 00000100:00100000:0.0:1761316073.693333:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322635200:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316074.205099:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316075.227889:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316076.251524:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316076.251562:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322635392:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761316076.251566:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316076.251573:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316076.251576:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316076.251581:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4318 00000100:00080000:2.0:1761316076.251585:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316076.251590:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316076.251614:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000008ee65757 x1846869322635584/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316076.251627:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316076.251631:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316076.251632:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316076.251637:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4318 00000100:00100000:0.0:1761316076.251639:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761316076.251641:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316076.251645:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316076.251657:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094d2894e x1846869322635648/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316076.251682:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322635776:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.251705:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322635456:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316076.251715:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316076.251742:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316076.251766:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635392 00000100:00100000:0.0:1761316076.251808:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3197 00000100:00100000:0.0:1761316076.251813:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322635392:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316076.251858:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322635392:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (224us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316076.251865:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3197 00000100:00100000:0.0:1761316076.251901:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322635392:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252101:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322635840:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252140:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316076.252173:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635776 00000100:00100000:2.0:1761316076.252204:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5127 00000100:00100000:2.0:1761316076.252209:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb727457 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322635776:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252245:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb727457 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322635776:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (531us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316076.252251:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5127 00000100:00100000:2.0:1761316076.252273:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635456 00000100:00100000:2.0:1761316076.252292:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5128 00000100:00100000:2.0:1761316076.252295:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322635456:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252326:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afcc50ba pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322635456:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (585us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316076.252335:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5128 00000100:00100000:2.0:1761316076.252375:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635840 00000100:00100000:2.0:1761316076.252396:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1648 00000100:00100000:2.0:1761316076.252399:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322635840:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252424:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322635840:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (285us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316076.252430:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1648 00000100:00100000:2.0:1761316076.252458:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cf3a4110 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322635456:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252468:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322635840:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.252592:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322635264:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.252628:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.252648:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322635520:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.252671:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.252680:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322635904:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.252698:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.252717:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1761316076.252722:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322635328:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.252745:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.252761:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322635712:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252777:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316076.252789:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316076.252789:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000041f26507 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322636032:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316076.252809:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322635968:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316076.252826:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.252845:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316076.252874:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635264 00000100:00100000:1.0:1761316076.252881:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635328 00000100:00100000:2.0:1761316076.252907:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1878 00000100:00100000:1.0:1761316076.252913:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635712 00000100:00100000:2.0:1761316076.252913:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322635264:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252952:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322635264:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (324us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316076.252960:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1878 00000100:00100000:1.0:1761316076.252964:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5129 00000100:00100000:1.0:1761316076.252970:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322635328:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.252995:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322635584:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253008:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a9c309 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322635328:12345-0@lo:400:kworker/u8:0.0 Request processed in 36us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316076.253014:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5129 00000100:00100000:2.0:1761316076.253028:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.253032:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5130 00000100:00100000:1.0:1761316076.253037:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322635712:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316076.253050:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322635648:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253070:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000084899189 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322635712:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316076.253077:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5130 00000100:00100000:2.0:1761316076.253083:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316076.253106:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635904 00000100:00100000:2.0:1761316076.253123:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635584 00000100:00100000:1.0:1761316076.253132:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635648 00000100:00100000:1.0:1761316076.253146:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1649 00000100:00100000:1.0:1761316076.253149:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322635904:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253175:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322635904:12345-0@lo:400:kworker/u8:0.0 Request processed in 25us (477us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316076.253180:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1649 00000100:00100000:1.0:1761316076.253190:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2525 00000100:00100000:1.0:1761316076.253192:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322635648:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316076.253286:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316076.253323:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322635648:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (242us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316076.253328:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2525 00000100:00100000:3.0:1761316076.253491:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635968 00000100:00100000:3.0:1761316076.253518:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1089 00000100:00100000:3.0:1761316076.253522:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322635968:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316076.253554:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316076.253572:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322635520 00000100:00100000:3.0:1761316076.253595:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322635968:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (751us total) trans 0 rc 0/0 00000100:00080000:1.0:1761316076.253598:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316076.253603:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316076.253603:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1089 00000100:00100000:1.0:1761316076.253609:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322635648:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316076.253636:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636032 00000100:00100000:1.0:1761316076.253643:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322635328:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253654:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000072c6c69d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322635712:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316076.253666:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3198 00000100:00100000:1.0:1761316076.253671:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322635968:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316076.253673:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322635520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253683:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003226e4c2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322635264:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316076.253694:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322635904:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316076.253716:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322635520:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (1043us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316076.253723:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3198 00000100:00100000:1.0:1761316076.253726:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322635520:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316076.253744:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3199 00000100:00100000:3.0:1761316076.253750:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322636032:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316076.253825:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2526 00000100:00100000:2.0:1761316076.253829:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322635584:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316076.253911:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316076.253942:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322635584:12345-0@lo:8:kworker/u8:0.0 Request processed in 112us (917us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316076.253947:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2526 00000100:00080000:2.0:1761316076.253968:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316076.253971:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316076.253974:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008ee65757 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322635584:0@lo:8:kworker/u8:0.0 00000004:00000024:3.0:1761316076.254314:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316076.254383:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322636032:12345-0@lo:41:osp-pre-0-1.0 Request processed in 633us (1558us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316076.254390:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3199 00000100:00100000:2.0:1761316076.255322:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000041f26507 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322636032:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316076.258919:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322635776:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761316077.279143:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316078.281928:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316078.281990:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316078.282035:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316078.282241:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580736 00000100:00100000:0.0:1761316078.282308:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580800 00000100:00100000:0.0:1761316078.282331:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5131 00000100:00100000:0.0:1761316078.282339:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307580736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316078.282491:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000014f5fd29 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307580736:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 152us (574us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316078.282499:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5131 00000100:00100000:0.0:1761316078.282529:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5132 00000100:00100000:0.0:1761316078.282534:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307580800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316078.282562:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000072c6c69d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307580800:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 28us (528us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316078.282573:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5132 00000100:00100000:2.0:1761316078.283299:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580672 00000100:00100000:2.0:1761316078.283383:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1879 00000100:00100000:2.0:1761316078.283389:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307580672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316078.283452:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307580672:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 62us (1462us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316078.283459:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1879 00000100:00100000:1.0:1761316078.286759:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316078.287095:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580864 00000100:00100000:3.0:1761316078.287159:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2527 00000100:00100000:3.0:1761316078.287167:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307580864:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316078.287274:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316078.287403:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307580864:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 235us (656us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316078.287412:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2527 00010000:00010000:1.0:1761316078.299706:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316078.811915:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322636096:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316078.812014:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316078.812115:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636096 00000100:00100000:0.0:1761316078.812250:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3200 00000100:00100000:0.0:1761316078.812258:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322636096:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316078.812305:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316078.813843:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e376567a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322636096:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1577us (1822us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316078.813858:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3200 00000100:00100000:2.0:1761316078.813995:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322636096:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316079.328759:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316080.349294:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761316081.372203:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316081.372211:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316081.372216:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316081.372227:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1761316081.372231:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316081.372237:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316081.372267:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b6d1e3fc x1846869322636544/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316081.372286:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316081.372289:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316081.372291:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316081.372294:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4323 00000100:00080000:3.0:1761316081.372296:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316081.372299:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316081.372311:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007c685a75 x1846869322636608/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316081.372414:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322636544:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316081.372527:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316081.372561:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322636608:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316081.372584:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316081.372634:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636544 00000100:00100000:3.0:1761316081.372669:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636608 00000100:00100000:3.0:1761316081.372688:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1761316081.372696:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322636544:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761316081.372791:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316081.372835:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322636544:12345-0@lo:8:kworker/u8:0.0 Request processed in 140us (338us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316081.372842:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2528 00000100:00100000:3.0:1761316081.372859:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2529 00000100:00100000:3.0:1761316081.372863:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322636608:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761316081.372937:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316081.372961:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322636608:12345-0@lo:8:kworker/u8:0.0 Request processed in 97us (377us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316081.372966:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2529 00000100:00080000:3.0:1761316081.373015:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316081.373019:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316081.373023:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322636544:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761316081.373033:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316081.373035:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316081.373037:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322636608:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373123:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322636288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373160:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316081.373191:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322636672:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373212:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316081.373228:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761316081.373234:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322636352:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373257:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316081.373269:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322636736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373287:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316081.373317:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636288 00000100:00100000:2.0:1761316081.373370:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636672 00000100:00100000:2.0:1761316081.373386:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636736 00000100:00100000:2.0:1761316081.373401:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5133 00000100:00100000:2.0:1761316081.373405:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322636288:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373473:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322636288:12345-0@lo:400:kworker/u8:0.0 Request processed in 67us (312us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316081.373480:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5133 00000100:00100000:2.0:1761316081.373493:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5134 00000100:00100000:2.0:1761316081.373508:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322636672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373536:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ee65757 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322636672:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (323us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316081.373541:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5134 00000100:00100000:2.0:1761316081.373550:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5135 00000100:00100000:2.0:1761316081.373552:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322636736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373576:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf3a4110 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322636736:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (289us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316081.373581:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5135 00000100:00100000:2.0:1761316081.373612:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322636736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373621:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002118a78f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322636288:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.373629:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322636672:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316081.375616:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322636224:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316081.375718:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316081.375776:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322636480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.375808:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636224 00000100:00100000:1.0:1761316081.375811:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322636160:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316081.375817:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316081.375847:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322636864:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.375861:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316081.375865:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1880 00000100:00100000:3.0:1761316081.375872:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322636224:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.375894:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322636416:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316081.375897:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316081.375940:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316081.375947:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322636224:12345-0@lo:400:kworker/u8:0.0 Request processed in 73us (229us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.375957:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1880 00000100:00100000:0.0:1761316081.375962:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322636224:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.375966:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322636800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.375997:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316081.376007:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636416 00000100:00100000:3.0:1761316081.376031:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636352 00000100:00100000:1.0:1761316081.376053:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5136 00000100:00100000:3.0:1761316081.376061:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636480 00000100:00100000:1.0:1761316081.376065:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000efeef3de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322636416:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.376072:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636160 00000100:00100000:3.0:1761316081.376086:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3201 00000100:00100000:3.0:1761316081.376091:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322636352:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.376111:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000efeef3de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322636416:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (172us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376138:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322636352:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (2874us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376145:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3201 00000100:00100000:3.0:1761316081.376157:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3202 00000100:00100000:3.0:1761316081.376161:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322636480:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.376187:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322636480:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (375us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376193:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3202 00000100:00100000:3.0:1761316081.376204:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3203 00000100:00100000:3.0:1761316081.376208:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322636160:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316081.376229:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000004:00000024:3.0:1761316081.376235:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316081.376242:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322636928:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316081.376265:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322636160:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (405us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376272:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3203 00000100:00100000:2.0:1761316081.376284:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316081.376293:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636864 00000100:00100000:2.0:1761316081.376323:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636928 00000100:00100000:3.0:1761316081.376324:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636800 00000100:00100000:3.0:1761316081.376369:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1650 00000100:00100000:3.0:1761316081.376374:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322636864:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.376389:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1090 00000100:00100000:2.0:1761316081.376395:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322636928:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316081.376405:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322636864:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (510us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376413:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1650 00002000:00000024:2.0:1761316081.376422:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316081.376428:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1651 00000100:00100000:3.0:1761316081.376434:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322636800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.376468:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322636800:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316081.376474:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322636928:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (189us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316081.376480:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1651 00000100:00100000:2.0:1761316081.376487:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1090 00000100:00100000:2.0:1761316081.376550:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322636928:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316081.376555:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322636352:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.376606:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322636480:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316081.376616:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322636864:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.377600:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000051c6d3bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322636160:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761316081.377611:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316081.377626:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1f55899 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322636416:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316081.377641:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322636800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316081.377965:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5136 00010000:00010000:0.0:1761316082.395645:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316083.399793:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316083.399848:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316083.400156:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316083.400886:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580992 00000100:00100000:1.0:1761316083.400980:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581056 00000100:00100000:1.0:1761316083.401005:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5137 00000100:00100000:1.0:1761316083.401013:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307580992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316083.401088:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5138 00000100:00100000:2.0:1761316083.401095:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307581056:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316083.401115:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f8a797b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307580992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 101us (1332us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316083.401124:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5137 00000100:00100000:2.0:1761316083.401135:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307581056:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 38us (985us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316083.401144:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5138 00000100:00100000:3.0:1761316083.401336:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307580928 00000100:00100000:3.0:1761316083.401596:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1881 00000100:00100000:3.0:1761316083.401601:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307580928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316083.401624:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316083.401643:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307580928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 41us (1795us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316083.401651:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1881 00000100:00100000:3.0:1761316083.401723:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581120 00000100:00100000:3.0:1761316083.402281:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2530 00000100:00100000:3.0:1761316083.402286:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307581120:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761316083.402420:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316083.402462:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307581120:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 175us (840us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316083.402468:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2530 00010000:00010000:3.0:1761316083.420145:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316083.931698:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316083.931731:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322636992:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316083.931822:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316083.932109:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322636992 00000100:00100000:3.0:1761316083.932173:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3204 00000100:00100000:3.0:1761316083.932180:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322636992:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316083.932224:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316083.932289:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322636992:12345-0@lo:41:osp-pre-1-0.0 Request processed in 109us (468us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316083.932297:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3204 00000100:00100000:1.0:1761316083.932442:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000049aba795 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322636992:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316084.443723:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316085.467736:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316086.491601:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637056:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316086.491710:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316086.491775:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316086.491780:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322637184:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761316086.491784:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316086.491790:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316086.491798:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4328 00000100:00080000:0.0:1761316086.491802:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316086.491809:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316086.491846:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000eeae700 x1846869322637440/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316086.491865:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316086.491868:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316086.491873:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316086.491876:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316086.491880:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4328 00000100:00080000:0.0:1761316086.491883:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316086.491888:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316086.491904:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c96a4a50 x1846869322637504/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316086.491914:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316086.491929:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322637248:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.491975:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316086.491986:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322637632:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492005:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322637568:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316086.492048:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316086.492124:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322637120:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316086.492162:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322637312:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.492177:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316086.492199:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316086.492211:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322637376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492213:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637248 00000100:00100000:3.0:1761316086.492251:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316086.492273:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322637760:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492289:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637632 00000100:00100000:3.0:1761316086.492306:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316086.492331:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5139 00000100:00100000:2.0:1761316086.492339:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322637248:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.492382:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637760 00000100:00100000:2.0:1761316086.492432:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000051c6d3bd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322637248:12345-0@lo:400:kworker/u8:2.0 Request processed in 92us (458us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.492441:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5139 00000100:00100000:2.0:1761316086.492470:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5140 00000100:00100000:2.0:1761316086.492475:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322637632:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492514:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000de3a181 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322637632:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (478us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.492521:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5140 00000100:00100000:2.0:1761316086.492569:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322637632:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.492599:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1652 00000100:00100000:3.0:1761316086.492609:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322637760:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492625:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637120 00000100:00100000:2.0:1761316086.492656:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1091 00000100:00100000:2.0:1761316086.492661:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322637120:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316086.492689:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316086.492703:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322637760:12345-0@lo:400:kworker/u8:2.0 Request processed in 92us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316086.492713:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1652 00000100:00100000:2.0:1761316086.492743:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322637120:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (566us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316086.492749:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637056 00000100:00100000:2.0:1761316086.492754:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1091 00000100:00100000:3.0:1761316086.492792:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637376 00000100:00100000:2.0:1761316086.492798:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322637440:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316086.492820:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3205 00000100:00100000:3.0:1761316086.492826:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322637056:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316086.492835:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:3.0:1761316086.492858:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316086.492866:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322637504:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316086.492922:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322637056:12345-0@lo:41:osp-pre-0-1.0 Request processed in 88us (1207us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.492925:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316086.492939:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3205 00000100:00100000:3.0:1761316086.492960:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3206 00000100:00100000:3.0:1761316086.492964:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322637376:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.492983:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637440 00000100:00100000:3.0:1761316086.493007:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322637376:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (756us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.493011:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637504 00000100:00100000:3.0:1761316086.493016:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3206 00000100:00100000:2.0:1761316086.493042:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2531 00000100:00100000:2.0:1761316086.493047:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322637440:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316086.493059:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2532 00000100:00100000:3.0:1761316086.493065:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322637504:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316086.493435:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316086.493485:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322637440:12345-0@lo:8:kworker/u8:2.0 Request processed in 437us (651us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316086.493492:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2531 00000100:00080000:2.0:1761316086.493524:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316086.493528:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316086.493530:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000eeae700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322637440:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316086.493569:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000960eacc4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322637376:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.493579:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001749d7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322637760:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.493592:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322637120:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316086.493644:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637184 00000100:00100000:2.0:1761316086.493668:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1882 00000100:00100000:2.0:1761316086.493672:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322637184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.493705:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322637184:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (1846us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.493711:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1882 00010000:00010000:2.0:1761316086.494723:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:02020000:3.0:1761316086.495093:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316086.495206:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322637504:12345-0@lo:8:kworker/u8:2.0 Request processed in 2138us (2288us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316086.495221:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2532 00000100:00100000:1.0:1761316086.495585:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637696:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316086.495595:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316086.495605:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316086.495612:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c96a4a50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322637504:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316086.495636:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316086.495689:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637056:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316086.495787:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637312 00000100:00100000:0.0:1761316086.495843:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5141 00000100:00100000:0.0:1761316086.495851:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322637312:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316086.495917:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0be819d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322637312:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (3719us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316086.495923:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5141 00000100:00100000:3.0:1761316086.496570:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316086.496634:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322637248:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.496646:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322637184:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.496689:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637696 00000100:00100000:3.0:1761316086.496733:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1653 00000100:00100000:3.0:1761316086.496739:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322637696:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.496800:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322637696:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (1168us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316086.496806:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1653 00000100:00100000:2.0:1761316086.496895:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637568 00000100:00100000:2.0:1761316086.496931:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5142 00000100:00100000:2.0:1761316086.496935:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322637568:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316086.496972:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1f55899 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322637568:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (407us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316086.496978:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5142 00000100:00100000:1.0:1761316086.497233:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322637312:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316086.497264:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637696:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316086.497431:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322637568:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761316087.517137:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316088.523889:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316088.523947:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316088.523975:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581184 00000100:00100000:2.0:1761316088.524037:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1883 00000100:00100000:2.0:1761316088.524046:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307581184:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316088.524060:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316088.524073:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581248 00000100:00100000:0.0:1761316088.524105:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316088.524123:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581312 00000100:00100000:2.0:1761316088.524124:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307581184:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 77us (242us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316088.524134:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1883 00000100:00100000:3.0:1761316088.524162:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5143 00000100:00100000:3.0:1761316088.524170:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307581248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316088.524194:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5144 00000100:00100000:2.0:1761316088.524208:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307581312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316088.524231:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c0153f06 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307581248:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 60us (285us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316088.524239:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5143 00000100:00100000:2.0:1761316088.524247:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001749d7de pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307581312:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 41us (188us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316088.524257:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5144 00000100:00100000:3.0:1761316088.524284:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581376 00000100:00100000:3.0:1761316088.524315:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2533 00000100:00100000:3.0:1761316088.524321:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ec6c998b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307581376:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761316088.524437:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316088.524492:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ec6c998b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307581376:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 170us (387us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316088.524500:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2533 00010000:00010000:1.0:1761316088.539583:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316089.051690:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316089.051714:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322637824:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316089.051778:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316089.051873:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637824 00000100:00100000:3.0:1761316089.051923:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3207 00000100:00100000:3.0:1761316089.051929:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322637824:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316089.051969:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316089.052025:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cbfcf664 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322637824:12345-0@lo:41:osp-pre-1-0.0 Request processed in 95us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316089.052031:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3207 00000100:00100000:1.0:1761316089.052226:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322637824:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316089.564174:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316090.588126:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316091.611660:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322637888:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316091.611768:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.611905:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637888 00000100:00100000:2.0:1761316091.611974:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3208 00000100:00100000:2.0:1761316091.611981:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322637888:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316091.612022:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316091.612079:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322637888:12345-0@lo:41:osp-pre-0-1.0 Request processed in 97us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316091.612086:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3208 00000100:00100000:2.0:1761316091.612134:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322637888:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316091.612799:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637952:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316091.612845:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761316091.614213:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316091.614222:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316091.614226:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316091.614234:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4334 00000100:00080000:2.0:1761316091.614238:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316091.614245:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316091.614269:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000034b089b1 x1846869322638272/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316091.614285:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316091.614288:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316091.614289:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316091.614292:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4334 00000100:00080000:2.0:1761316091.614294:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316091.614296:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316091.614304:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a1edac59 x1846869322638336/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316091.614361:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322638272:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614419:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614453:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322638336:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614473:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614501:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322638016:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614531:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614542:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322638400:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614561:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614577:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316091.614581:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614604:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614638:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638400 00000100:00100000:2.0:1761316091.614679:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5145 00000100:00100000:2.0:1761316091.614685:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322638400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614733:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000eeae700 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322638400:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (171us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316091.614740:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5145 00000100:00100000:2.0:1761316091.614778:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb399f09 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322638400:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614807:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322638144:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614826:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614837:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322638528:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614853:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614866:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761316091.614870:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322638080:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614886:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614896:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322638464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.614911:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316091.614933:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638272 00000100:00100000:2.0:1761316091.614953:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638336 00000100:00100000:2.0:1761316091.614975:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638528 00000100:00100000:2.0:1761316091.614995:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1654 00000100:00100000:2.0:1761316091.615000:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322638528:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615020:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322637952 00000100:00100000:2.0:1761316091.615032:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322638528:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (177us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316091.615039:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1654 00000100:00100000:2.0:1761316091.615053:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2534 00000100:00100000:2.0:1761316091.615060:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322638272:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615102:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1092 00000100:00100000:3.0:1761316091.615110:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322637952:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316091.615144:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00010000:02020000:2.0:1761316091.615168:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316091.615222:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322638272:12345-0@lo:8:kworker/u8:0.0 Request processed in 164us (803us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316091.615231:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2534 00000100:00100000:3.0:1761316091.615242:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322637952:12345-0@lo:13:osp-pre-1-1.0 Request processed in 128us (2393us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316091.615252:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2535 00000100:00100000:2.0:1761316091.615259:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322638336:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615267:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1092 00000100:00100000:3.0:1761316091.615306:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638016 00000100:00100000:3.0:1761316091.615335:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1884 00010000:02020000:2.0:1761316091.615340:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316091.615389:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322638016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.615408:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322638336:12345-0@lo:8:kworker/u8:0.0 Request processed in 147us (932us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316091.615417:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2535 00000100:00100000:3.0:1761316091.615466:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322638016:12345-0@lo:400:kworker/u8:0.0 Request processed in 122us (931us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316091.615478:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1884 00000100:00080000:2.0:1761316091.615487:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316091.615493:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316091.615499:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322638272:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761316091.615516:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316091.615520:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316091.615526:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322638336:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615528:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638144 00000100:00100000:3.0:1761316091.615567:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638080 00000100:00100000:2.0:1761316091.615589:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322637952:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316091.615594:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638464 00000100:00100000:3.0:1761316091.615624:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5146 00000100:00100000:3.0:1761316091.615634:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322638144:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316091.615642:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5147 00000100:00100000:2.0:1761316091.615651:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322638080:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615691:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb399f09 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322638144:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (861us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316091.615702:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5146 00000100:00100000:2.0:1761316091.615713:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0db494f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322638080:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (822us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316091.615725:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5148 00000100:00100000:3.0:1761316091.615732:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322638464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316091.615772:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf11a655 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322638464:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (860us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316091.616265:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638208 00000100:00100000:0.0:1761316091.616311:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3209 00000100:00100000:0.0:1761316091.616316:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616385:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638208:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (1779us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316091.616392:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3209 00000100:00100000:0.0:1761316091.616499:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638208:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616511:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322638016:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616573:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ba62614 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322638080:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616585:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322638464:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616596:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b59cfdd8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322638144:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316091.616607:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322638528:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761316091.616720:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316091.618550:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5147 00000100:00100000:3.0:1761316091.618698:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5148 00010000:00010000:2.0:1761316092.635644:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316093.645322:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316093.645400:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316093.645445:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316093.645523:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581568 00000100:00100000:2.0:1761316093.645602:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581504 00000100:00100000:2.0:1761316093.645628:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5149 00000100:00100000:2.0:1761316093.645637:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307581568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316093.645744:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000422f974e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307581568:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 108us (345us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316093.645753:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5149 00000100:00100000:2.0:1761316093.645773:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5150 00000100:00100000:2.0:1761316093.645778:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307581504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316093.645803:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000060d3b1bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307581504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (359us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316093.645811:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5150 00000100:00100000:3.0:1761316093.646520:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316093.646576:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581632 00000100:00100000:1.0:1761316093.646663:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2536 00000100:00100000:1.0:1761316093.646669:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307581632:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316093.646770:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316093.646830:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307581632:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 160us (313us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316093.646838:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2536 00000100:00100000:3.0:1761316093.647489:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581440 00000100:00100000:3.0:1761316093.647538:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1885 00000100:00100000:3.0:1761316093.647544:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307581440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316093.647603:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307581440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 58us (2295us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316093.647610:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1885 00010000:00010000:3.0:1761316093.660487:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316094.171913:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316094.171945:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638592:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316094.172039:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316094.172226:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638592 00000100:00100000:1.0:1761316094.172287:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3210 00000100:00100000:1.0:1761316094.172294:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638592:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316094.172337:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316094.172426:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bb94dc5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638592:12345-0@lo:41:osp-pre-1-0.0 Request processed in 131us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316094.172434:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3210 00000100:00100000:2.0:1761316094.172521:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638592:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316094.684325:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316095.708028:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316096.731694:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316096.731704:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316096.731711:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316096.731722:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4339 00000100:00080000:2.0:1761316096.731727:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316096.731734:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316096.731766:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000828d143a x1846869322638976/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316096.731784:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316096.731787:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316096.731788:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316096.731792:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4339 00000100:00080000:2.0:1761316096.731794:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316096.731797:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316096.731811:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b46f99e6 x1846869322639040/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316096.731908:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322638656:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732001:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732036:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638912:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732065:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732081:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322639296:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732103:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732128:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761316096.732134:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322638848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732160:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732175:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322639232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732195:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732222:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322638720:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732247:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732260:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322639104:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732280:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732295:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761316096.732301:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322638784:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732322:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732334:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322639168:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732402:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732429:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322638976:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732454:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732469:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322639040:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732502:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316096.732545:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638656 00000100:00100000:3.0:1761316096.732584:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638848 00000100:00100000:2.0:1761316096.732590:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1886 00000100:00100000:2.0:1761316096.732597:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322638656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316096.732627:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638720 00000100:00100000:3.0:1761316096.732640:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639104 00000100:00100000:3.0:1761316096.732650:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639168 00000100:00100000:2.0:1761316096.732661:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322638656:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (662us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316096.732662:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5151 00000100:00100000:3.0:1761316096.732665:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322638848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732669:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1886 00000100:00100000:3.0:1761316096.732700:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5b00f51 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322638848:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (540us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316096.732704:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639296 00000100:00100000:3.0:1761316096.732706:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5151 00000100:00100000:3.0:1761316096.732717:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5152 00000100:00100000:3.0:1761316096.732720:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003d22da5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322638720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732726:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639232 00000100:00100000:2.0:1761316096.732740:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638976 00000100:00100000:3.0:1761316096.732742:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003d22da5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322638720:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (496us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316096.732748:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5152 00000100:00100000:2.0:1761316096.732749:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639040 00000100:00100000:3.0:1761316096.732756:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5153 00000100:00100000:2.0:1761316096.732758:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1655 00000100:00100000:3.0:1761316096.732759:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322639104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732761:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322639296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316096.732780:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087f8a647 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322639104:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (499us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316096.732786:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322639296:12345-0@lo:400:kworker/u8:0.0 Request processed in 24us (683us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316096.732786:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5153 00000100:00100000:2.0:1761316096.732792:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1655 00000100:00100000:3.0:1761316096.732795:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5154 00000100:00100000:3.0:1761316096.732797:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322639168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316096.732800:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1656 00000100:00100000:2.0:1761316096.732803:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322639232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316096.732819:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee2d2bc1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322639168:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (417us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316096.732820:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322639232:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (625us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316096.732825:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1656 00000100:00100000:2.0:1761316096.732834:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2537 00000100:00100000:2.0:1761316096.732837:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322638976:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316096.732840:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5154 00000100:00100000:3.0:1761316096.732858:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2538 00000100:00100000:3.0:1761316096.732861:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322639040:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316096.732996:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316096.733002:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316096.733108:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638912 00000100:00100000:1.0:1761316096.733141:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322638784 00000100:00100000:1.0:1761316096.733159:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3211 00000100:00100000:1.0:1761316096.733163:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638912:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316096.733201:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e53cb60 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322638912:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (1136us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316096.733206:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3211 00000100:00100000:1.0:1761316096.733218:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3212 00000100:00100000:1.0:1761316096.733221:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322638784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316096.733242:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322638784:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (921us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316096.733248:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3212 00000100:00100000:3.0:1761316096.733251:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322639040:12345-0@lo:8:kworker/u8:0.0 Request processed in 389us (750us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316096.733257:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2538 00000100:00100000:3.0:1761316096.733281:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322639360:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316096.733310:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316096.733369:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dad31777 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322638784:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316096.733380:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1761316096.733381:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322639168:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316096.733384:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316096.733399:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322638976:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316096.733403:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000604db0f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322638720:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316096.733408:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316096.733411:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316096.733412:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322639104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316096.733413:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322639040:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316096.733444:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639360 00000100:00100000:1.0:1761316096.733445:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a1edac59 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322638848:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316096.733453:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322639232:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316096.733461:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322638656:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316096.733468:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1093 00000100:00100000:1.0:1761316096.733469:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322638912:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316096.733472:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322639360:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316096.733478:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322639296:0@lo:400:kworker/u8:0.0 00002000:00000024:3.0:1761316096.733495:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316096.733527:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322639360:12345-0@lo:13:osp-pre-1-1.0 Request processed in 55us (217us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316096.733534:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1093 00000100:00100000:1.0:1761316096.733545:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322639360:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316096.734442:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322638976:12345-0@lo:8:kworker/u8:0.0 Request processed in 1597us (1982us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316096.734453:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2537 00000100:00100000:1.0:1761316096.735229:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322639424:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316096.735296:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761316096.735448:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316096.737435:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639424 00000100:00100000:0.0:1761316096.737532:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3213 00000100:00100000:0.0:1761316096.737540:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322639424:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316096.737587:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316096.737668:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322639424:12345-0@lo:41:osp-pre-0-1.0 Request processed in 128us (2372us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316096.737677:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3213 00000100:00100000:1.0:1761316096.737902:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006fc130f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322639424:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761316097.757984:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316098.763500:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316098.765569:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581760 00000100:00100000:2.0:1761316098.765674:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5155 00000100:00100000:2.0:1761316098.765684:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307581760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316098.765808:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307581760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 125us (2316us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316098.765818:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5155 00000100:00100000:3.0:1761316098.769365:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316098.769506:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581696 00000100:00100000:1.0:1761316098.769597:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1887 00000100:00100000:1.0:1761316098.769605:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307581696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316098.769699:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307581696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 94us (357us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316098.769707:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1887 00000100:00100000:3.0:1761316098.774713:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316098.774761:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316098.774911:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581824 00000100:00100000:1.0:1761316098.774974:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5156 00000100:00100000:1.0:1761316098.774981:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307581824:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316098.775050:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e65d9e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307581824:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 68us (289us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316098.775058:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5156 00000100:00100000:2.0:1761316098.775675:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581888 00000100:00100000:2.0:1761316098.775708:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2539 00000100:00100000:2.0:1761316098.775712:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307581888:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761316098.775805:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316098.775837:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307581888:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 123us (1131us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316098.775845:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2539 00010000:00010000:2.0:1761316098.781977:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316099.291662:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322639488:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316099.291731:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316099.291839:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639488 00000100:00100000:3.0:1761316099.291889:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3214 00000100:00100000:3.0:1761316099.291896:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322639488:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316099.291928:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316099.291980:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322639488:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (248us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316099.291987:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3214 00000100:00100000:1.0:1761316099.292071:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322639488:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316099.803777:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316100.827639:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316101.851539:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322639552:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316101.851586:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316101.851615:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316101.851694:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639552 00000100:00100000:0.0:1761316101.851740:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3215 00000100:00100000:0.0:1761316101.851746:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322639552:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316101.851776:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316101.851831:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000719d773a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322639552:12345-0@lo:41:osp-pre-0-1.0 Request processed in 84us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316101.851840:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3215 00000100:00100000:3.0:1761316101.851860:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316101.851874:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322639616:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316101.851907:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316101.851939:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639616 00000100:00100000:3.0:1761316101.851966:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1094 00000100:00100000:3.0:1761316101.851970:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322639616:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316101.851990:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316101.852034:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322639616:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (127us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316101.852041:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1094 00000100:00100000:1.0:1761316101.852109:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322639552:0@lo:41:osp-pre-0-1.0 00000100:00080000:1.0:1761316101.852199:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316101.852203:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322639744:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316101.852206:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316101.852210:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316101.852222:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4344 00000100:00080000:1.0:1761316101.852225:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316101.852229:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316101.852243:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316101.852251:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f2b515d3 x1846869322639936/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316101.852269:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316101.852269:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:1.0:1761316101.852275:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316101.852276:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316101.852278:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322639808:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316101.852282:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4344 00000100:00080000:1.0:1761316101.852285:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316101.852288:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316101.852302:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004be373e1 x1846869322640000/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316101.852311:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316101.852340:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322640128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852384:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322639936:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316101.852402:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316101.852425:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316101.852428:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316101.852433:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322640192:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852451:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322640000:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316101.852472:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316101.852491:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639744 00000100:00100000:2.0:1761316101.852512:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639936 00000100:00100000:1.0:1761316101.852524:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639808 00000100:00100000:2.0:1761316101.852540:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640192 00000100:00100000:1.0:1761316101.852551:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640128 00000100:00100000:2.0:1761316101.852569:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1657 00000100:00100000:2.0:1761316101.852575:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322640192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852582:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5157 00000100:00100000:1.0:1761316101.852588:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f2449da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322639744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316101.852612:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322640192:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (141us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316101.852619:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1657 00000100:00100000:1.0:1761316101.852632:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f2449da pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322639744:12345-0@lo:400:kworker/u8:0.0 Request processed in 43us (389us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316101.852638:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2540 00000100:00100000:1.0:1761316101.852644:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5157 00000100:00100000:2.0:1761316101.852646:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322639936:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852662:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5158 00000100:00100000:1.0:1761316101.852666:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322639808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852690:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f0d14a4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322639808:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (384us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316101.852696:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5158 00000100:00100000:1.0:1761316101.852707:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5159 00000100:00100000:1.0:1761316101.852710:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322640128:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761316101.852735:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316101.852736:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a3db8e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322640128:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316101.852745:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5159 00000100:00100000:2.0:1761316101.852772:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322639936:12345-0@lo:8:kworker/u8:0.0 Request processed in 126us (346us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316101.852779:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2540 00000100:00100000:1.0:1761316101.852794:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316101.852814:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322640192:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316101.852827:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761316101.852831:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322640128:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316101.852833:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316101.852840:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2b515d3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322639936:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316101.852844:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000828d143a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322639808:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316101.852860:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b46f99e6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322639744:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316101.852865:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640000 00000100:00100000:1.0:1761316101.852888:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2541 00000100:00100000:1.0:1761316101.852892:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322640000:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316101.852968:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316101.853009:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322640000:12345-0@lo:8:kworker/u8:0.0 Request processed in 116us (218us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316101.853015:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2541 00000100:00100000:0.0:1761316101.853129:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322639680:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853163:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316101.853179:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322640064:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853198:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316101.853222:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322639616:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316101.853236:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316101.853240:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322639872:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853262:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316101.853285:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639680 00000100:00100000:0.0:1761316101.853307:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1888 00000100:00100000:0.0:1761316101.853310:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322639680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853340:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322639680:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316101.853377:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1888 00000100:00100000:0.0:1761316101.853403:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640064 00000100:00100000:0.0:1761316101.853425:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5160 00000100:00100000:0.0:1761316101.853428:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322640064:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853457:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322640064:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316101.853463:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5160 00000100:00100000:0.0:1761316101.853484:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c685a75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322639680:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853493:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322640064:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853513:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322639872 00000100:00100000:0.0:1761316101.853532:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3216 00000100:00100000:0.0:1761316101.853535:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322639872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316101.853564:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322639872:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (302us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316101.853569:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3216 00000100:00100000:0.0:1761316101.853591:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322639872:0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761316101.854606:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316101.854614:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316101.854618:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322640000:0@lo:8:kworker/u8:0.0 00010000:00010000:3.0:1761316102.875706:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316103.880252:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316103.880296:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316103.880434:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307581952 00000100:00100000:1.0:1761316103.880477:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1889 00000100:00100000:1.0:1761316103.880483:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307581952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316103.880549:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307581952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316103.880560:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1889 00000100:00100000:2.0:1761316103.882504:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582144 00000100:00100000:2.0:1761316103.882562:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2542 00000100:00100000:2.0:1761316103.882568:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307582144:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761316103.882654:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00010000:02020000:2.0:1761316103.882674:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316103.882783:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582016 00000100:00100000:2.0:1761316103.882842:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307582144:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 272us (2545us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316103.882851:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2542 00000100:00100000:3.0:1761316103.883117:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316103.883512:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582080 00000100:00100000:0.0:1761316103.885864:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5161 00000100:00100000:0.0:1761316103.885883:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307582016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316103.885964:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000604db0f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307582016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 86us (3313us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316103.886022:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5162 00000100:00100000:2.0:1761316103.886028:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316103.886059:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dad31777 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582080:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (2944us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316103.886066:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5162 00000100:00100000:0.0:1761316103.890504:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5161 00010000:00010000:3.0:1761316103.899715:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316104.411683:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322640256:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316104.411755:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316104.413387:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640256 00000100:00100000:2.0:1761316104.413451:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3217 00000100:00100000:2.0:1761316104.413457:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322640256:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316104.413503:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316104.413838:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004be373e1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322640256:12345-0@lo:41:osp-pre-1-0.0 Request processed in 378us (2082us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316104.413846:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3217 00000100:00100000:0.0:1761316104.414192:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322640256:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316104.924821:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316105.948413:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316106.971793:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316106.971835:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316106.971853:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322640384:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316106.971912:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316106.972178:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322640320:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316106.972211:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316106.972368:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316106.972375:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316106.972379:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316106.972384:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4349 00000100:00080000:3.0:1761316106.972387:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316106.972392:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316106.972412:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000956eaacd x1846869322640704/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316106.972425:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316106.972428:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316106.972429:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316106.972432:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4349 00000100:00080000:3.0:1761316106.972434:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316106.972437:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316106.972446:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005e264e5f x1846869322640768/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316106.972490:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640320 00000100:00100000:3.0:1761316106.972503:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322640448:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316106.972535:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1761316106.972540:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316106.972544:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322640320:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316106.972561:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322640832:0@lo:400:kworker/u8:2.0 00002000:00000024:1.0:1761316106.972573:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316106.972591:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316106.972614:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316106.972620:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322640320:12345-0@lo:13:osp-pre-1-1.0 Request processed in 76us (407us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316106.972622:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322640640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316106.972633:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1095 00000100:00100000:3.0:1761316106.972655:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316106.972660:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640448 00000100:00100000:3.0:1761316106.972690:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640384 00000100:00100000:1.0:1761316106.972693:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1890 00000100:00100000:1.0:1761316106.972697:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322640448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316106.972725:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640640 00000100:00100000:1.0:1761316106.972736:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322640448:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (196us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316106.972744:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1890 00000100:00100000:3.0:1761316106.972749:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3218 00000100:00100000:3.0:1761316106.972755:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322640384:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316106.972779:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316106.972781:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3219 00000100:00100000:1.0:1761316106.972787:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322640640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316106.972820:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322640384:12345-0@lo:41:osp-pre-0-1.0 Request processed in 63us (907us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316106.972832:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322640640:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (175us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316106.972844:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3218 00000100:00100000:0.0:1761316106.973086:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322640576:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973112:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316106.973127:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322640960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973145:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316106.973176:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069497e7a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322640320:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316106.973205:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761316106.973210:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322640512:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973235:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316106.973255:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322640896:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973274:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316106.973296:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640832 00000100:00100000:0.0:1761316106.973333:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640576 00000100:00100000:0.0:1761316106.973370:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640512 00000100:00100000:0.0:1761316106.973381:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640896 00000100:00100000:0.0:1761316106.973392:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5163 00000100:00100000:0.0:1761316106.973396:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322640832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973423:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322640832:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (834us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316106.973428:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5163 00000100:00100000:0.0:1761316106.973438:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5164 00000100:00100000:0.0:1761316106.973441:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322640576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.973462:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007c685a75 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322640576:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316106.973470:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5164 00000100:00100000:0.0:1761316106.973479:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5165 00000100:00100000:0.0:1761316106.973482:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069497e7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322640512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316106.973577:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000956eaacd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322640704:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316106.973609:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316106.973624:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322640768:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316106.973641:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316106.973665:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5166 00000100:00100000:2.0:1761316106.973668:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322640896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316106.973693:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef3df022 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322640896:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (419us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316106.973699:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5166 00000100:00100000:2.0:1761316106.973717:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640960 00000100:00100000:2.0:1761316106.973737:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640704 00000100:00100000:2.0:1761316106.973749:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322640768 00000100:00100000:2.0:1761316106.973761:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1658 00000100:00100000:2.0:1761316106.973764:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322640960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316106.973789:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322640960:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (643us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316106.973794:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1658 00000100:00100000:2.0:1761316106.973804:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2543 00000100:00100000:2.0:1761316106.973806:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322640704:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316106.973904:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316106.974069:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069497e7a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322640512:12345-0@lo:400:kworker/u8:2.0 Request processed in 586us (834us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316106.974076:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5165 00000100:00100000:0.0:1761316106.974124:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322640512:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974134:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322640896:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974142:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322640576:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974149:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322640960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974193:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322640640:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974201:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322640448:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974208:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322640832:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316106.974222:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f595e60f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322640384:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316106.974257:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2544 00000100:00100000:0.0:1761316106.974260:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322640768:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761316106.974360:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316106.974390:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322640768:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (749us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316106.974397:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2544 00000100:00080000:0.0:1761316106.974455:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316106.974458:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316106.974461:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322640768:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316106.974824:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322640704:12345-0@lo:8:kworker/u8:2.0 Request processed in 1016us (1214us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316106.974843:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2543 00000100:00100000:1.0:1761316106.975103:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3219 00000100:00080000:0.0:1761316106.975246:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316106.975251:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316106.975254:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000956eaacd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322640704:0@lo:8:kworker/u8:2.0 00010000:00010000:0.0:1761316107.995765:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316109.004127:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316109.004196:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316109.004243:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316109.004281:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316109.004387:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582400 00000100:00100000:2.0:1761316109.004435:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2545 00000100:00100000:2.0:1761316109.004442:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307582400:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316109.004537:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316109.004589:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307582400:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 148us (471us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316109.004597:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2545 00000100:00100000:3.0:1761316109.004793:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582208 00000100:00100000:3.0:1761316109.004830:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1891 00000100:00100000:3.0:1761316109.004834:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307582208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316109.004884:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307582208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 48us (640us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316109.004891:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1891 00000100:00100000:0.0:1761316109.006130:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582272 00000100:00100000:0.0:1761316109.006174:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582336 00000100:00100000:0.0:1761316109.006322:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5167 00000100:00100000:0.0:1761316109.006329:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000179ae6e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307582272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316109.006517:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5168 00000100:00100000:2.0:1761316109.006523:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf642378 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582336:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316109.006570:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf642378 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582336:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 47us (2289us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316109.006577:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5168 00000100:00100000:0.0:1761316109.007012:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000179ae6e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307582272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 682us (2815us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316109.007020:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5167 00010000:00010000:3.0:1761316109.021870:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316109.531625:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000018d8e118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322641024:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316109.531697:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316109.532463:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641024 00000100:00100000:1.0:1761316109.532972:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3220 00000100:00100000:1.0:1761316109.532979:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322641024:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316109.533012:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316109.535117:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001198e496 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322641024:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2131us (3414us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316109.535132:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3220 00000100:00100000:2.0:1761316109.535318:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000018d8e118 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322641024:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316110.043852:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316111.067501:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316112.091532:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000166a802b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322641088:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1761316112.091560:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316112.091602:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316112.091647:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316112.091652:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316112.091656:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316112.091661:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4354 00000100:00080000:1.0:1761316112.091664:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316112.091669:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316112.091687:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006cf388c6 x1846869322641472/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316112.091702:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316112.091704:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316112.091706:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316112.091708:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4354 00000100:00080000:1.0:1761316112.091710:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316112.091713:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316112.091723:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f1a5999a x1846869322641536/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316112.091733:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322641152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.091783:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316112.091806:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322641216:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.091812:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322641408:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.091847:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316112.091852:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316112.091875:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001418520e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322641600:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.091888:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1f6a7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322641792:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.091899:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316112.091917:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f887dab3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322641856:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316112.091924:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316112.091946:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316112.091971:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641152 00000100:00100000:1.0:1761316112.091987:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006cf388c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322641472:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092011:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316112.092014:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1892 00000100:00100000:3.0:1761316112.092023:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322641152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092027:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322641536:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092049:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316112.092080:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322641152:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (296us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316112.092083:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641216 00000100:00100000:3.0:1761316112.092089:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1892 00000100:00100000:1.0:1761316112.092113:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641600 00000100:00100000:3.0:1761316112.092128:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641408 00000100:00100000:1.0:1761316112.092136:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5169 00000100:00100000:1.0:1761316112.092141:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d33a9583 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322641216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092160:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641856 00000100:00100000:1.0:1761316112.092184:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d33a9583 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322641216:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.092188:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3221 00000100:00100000:1.0:1761316112.092199:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5169 00000100:00100000:3.0:1761316112.092199:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322641408:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092220:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5170 00000100:00100000:1.0:1761316112.092224:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005da9e842 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322641600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092237:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322641408:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.092244:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3221 00000100:00100000:1.0:1761316112.092263:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005da9e842 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322641600:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (362us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.092264:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3222 00000100:00100000:3.0:1761316112.092272:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003d6c654 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322641856:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316112.092275:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5170 00000004:00000024:3.0:1761316112.092309:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316112.092365:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003d6c654 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322641856:12345-0@lo:41:osp-pre-0-1.0 Request processed in 70us (397us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.092374:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3222 00000100:00100000:1.0:1761316112.092397:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322641216:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092400:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641792 00000100:00100000:1.0:1761316112.092409:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001418520e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322641600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092425:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f887dab3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322641856:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316112.092429:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641472 00000100:00100000:3.0:1761316112.092448:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641536 00000100:00100000:3.0:1761316112.092465:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1659 00000100:00100000:3.0:1761316112.092470:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322641792:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316112.092477:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2546 00000100:00100000:1.0:1761316112.092483:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322641472:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092501:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322641792:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (579us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.092508:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1659 00000100:00100000:3.0:1761316112.092520:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2547 00000100:00100000:3.0:1761316112.092524:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322641536:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316112.092662:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316112.092694:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316112.092725:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322641536:12345-0@lo:8:kworker/u8:2.0 Request processed in 200us (677us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316112.092731:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2547 00000100:00080000:3.0:1761316112.092771:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316112.092775:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316112.092778:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322641536:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092801:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1f6a7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322641792:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092809:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322641152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.092816:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322641408:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.098681:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070eedddd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641280:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.098760:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316112.098806:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322641664:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.098852:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316112.098880:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761316112.098886:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e341f863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322641344:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.098916:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316112.098933:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c74a1dc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641728:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.098952:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316112.098980:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641728 00000100:00100000:2.0:1761316112.099014:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1660 00000100:00100000:2.0:1761316112.099019:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd918d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322641728:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.099063:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd918d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322641728:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (110us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316112.099069:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1660 00000100:00100000:2.0:1761316112.099089:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641664 00000100:00100000:2.0:1761316112.099108:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641344 00000100:00100000:2.0:1761316112.099122:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5171 00000100:00100000:2.0:1761316112.099125:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af404214 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322641664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.099151:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af404214 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322641664:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (299us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316112.099156:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5171 00000100:00100000:2.0:1761316112.099165:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5172 00000100:00100000:2.0:1761316112.099168:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8dd11c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322641344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.099187:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8dd11c1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322641344:12345-0@lo:400:kworker/u8:2.0 Request processed in 18us (272us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316112.099192:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5172 00000100:00100000:2.0:1761316112.099230:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e341f863 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322641344:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.099240:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c74a1dc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641728:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316112.099247:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322641664:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.099366:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641280 00000100:00100000:3.0:1761316112.099402:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3223 00000100:00100000:3.0:1761316112.099406:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322641280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316112.099440:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000008d853e9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322641280:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (681us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316112.099445:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3223 00000100:00100000:2.0:1761316112.099519:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070eedddd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641280:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316112.099925:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641088 00000100:00100000:0.0:1761316112.099956:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1096 00000100:00100000:0.0:1761316112.099960:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322641088:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316112.099982:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316112.100031:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322641088:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (8429us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316112.100036:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1096 00000100:00100000:3.0:1761316112.100784:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000166a802b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322641088:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316112.119672:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316112.119680:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316112.119686:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006cf388c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322641472:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316112.121162:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322641472:12345-0@lo:8:kworker/u8:2.0 Request processed in 28673us (29146us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316112.121174:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2546 00010000:00010000:1.0:1761316113.115781:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316114.122636:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316114.122682:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316114.122715:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316114.122756:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316114.122791:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582592 00000100:00100000:2.0:1761316114.122792:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582528 00000100:00100000:0.0:1761316114.122846:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5173 00000100:00100000:0.0:1761316114.122853:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c24f861b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316114.122866:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582464 00000100:00100000:3.0:1761316114.122904:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1893 00000100:00100000:3.0:1761316114.122910:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307582464:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316114.122927:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c24f861b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307582592:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 73us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316114.122935:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5173 00000100:00100000:3.0:1761316114.122949:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307582464:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 38us (320us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316114.122963:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1893 00000100:00100000:1.0:1761316114.123171:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582656 00000100:00100000:1.0:1761316114.123202:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2548 00000100:00100000:1.0:1761316114.123206:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b99c8f0c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307582656:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316114.123321:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316114.123392:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b99c8f0c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307582656:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 185us (710us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316114.123399:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2548 00000100:00100000:2.0:1761316114.123560:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5174 00000100:00100000:2.0:1761316114.123565:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000006746eb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307582528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316114.123605:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000006746eb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307582528:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 39us (850us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316114.123611:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5174 00010000:00010000:0.0:1761316114.140382:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316114.651880:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316114.652636:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000166a802b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322641920:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316114.652708:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316114.652833:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641920 00000100:00100000:2.0:1761316114.652885:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3224 00000100:00100000:2.0:1761316114.652891:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f887dab3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322641920:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316114.652925:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316114.652975:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f887dab3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322641920:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (267us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316114.652982:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3224 00000100:00100000:1.0:1761316114.653092:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000166a802b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322641920:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316115.163591:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316116.190204:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316117.211497:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316117.211513:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001418520e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641984:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316117.211578:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.211687:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322641984 00000100:00100000:1.0:1761316117.211737:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3225 00000100:00100000:2.0:1761316117.211737:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322642048:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316117.211745:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000255f7186 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322641984:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316117.211771:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:1.0:1761316117.211781:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316117.211829:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000255f7186 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322641984:12345-0@lo:41:osp-pre-0-1.0 Request processed in 84us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316117.211837:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3225 00000100:00100000:1.0:1761316117.211862:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322642112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316117.211888:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.211908:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316117.211914:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322642304:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316117.211933:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.211965:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642112 00000100:00100000:2.0:1761316117.211968:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322642176:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.211973:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642048 00000100:00100000:1.0:1761316117.211999:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1097 00000100:00100000:2.0:1761316117.212006:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.212007:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322642112:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316117.212011:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1894 00000100:00100000:3.0:1761316117.212018:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322642048:12345-0@lo:400:kworker/u8:0.0 00002000:00000024:1.0:1761316117.212032:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316117.212047:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001418520e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322641984:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316117.212064:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1761316117.212084:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316117.212089:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316117.212092:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316117.212099:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4359 00000100:00080000:0.0:1761316117.212101:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316117.212104:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316117.212124:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d93c9bea x1846869322642368/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316117.212139:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316117.212143:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316117.212144:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316117.212147:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4359 00000100:00080000:0.0:1761316117.212148:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316117.212151:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316117.212161:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005fdf8940 x1846869322642432/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316117.212280:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000039b44fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322642624:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316117.212325:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322642560:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316117.213527:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316117.213528:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316117.213535:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322642048:12345-0@lo:400:kworker/u8:0.0 Request processed in 1512us (1760us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316117.213536:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322642112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1526us (1643us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316117.213557:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1097 00000100:00100000:3.0:1761316117.213557:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1894 00000100:00100000:2.0:1761316117.213569:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322642240:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316117.213591:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322642368:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316117.213606:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.213608:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322642496:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.213617:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642304 00000100:00100000:0.0:1761316117.213627:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.213649:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316117.213657:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322642432:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316117.213658:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3226 00000100:00100000:3.0:1761316117.213669:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7a799d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322642304:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316117.213675:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642176 00000100:00100000:0.0:1761316117.213692:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316117.213698:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322642112:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316117.213724:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642560 00000100:00100000:3.0:1761316117.213727:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7a799d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322642304:12345-0@lo:400:kworker/u8:0.0 Request processed in 58us (1791us total) trans 0 rc 0/0 00010000:00010000:0.0:1761316117.213733:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316117.213734:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322642304:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.213738:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3226 00000100:00100000:2.0:1761316117.213748:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642240 00000100:00100000:2.0:1761316117.213765:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642496 00000100:00100000:0.0:1761316117.213776:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642624 00000100:00100000:1.0:1761316117.213776:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5175 00000100:00100000:1.0:1761316117.213782:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ef889cf0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322642176:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.213782:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642368 00000100:00100000:2.0:1761316117.213794:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5176 00000100:00100000:2.0:1761316117.213799:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e63e38d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322642560:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.213811:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642432 00000100:00100000:0.0:1761316117.213823:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1661 00000100:00100000:0.0:1761316117.213833:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322642624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316117.213833:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ef889cf0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322642176:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (1826us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316117.213841:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e63e38d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322642560:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316117.213846:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2549 00000100:00100000:1.0:1761316117.213848:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5175 00000100:00100000:3.0:1761316117.213855:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322642368:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316117.213856:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5176 00000100:00100000:1.0:1761316117.213871:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5177 00000100:00100000:1.0:1761316117.213877:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d05b7376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322642240:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316117.213882:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5178 00000100:00100000:0.0:1761316117.213885:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322642624:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (367us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316117.213892:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f937f1e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322642496:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316117.213897:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1661 00000100:00100000:1.0:1761316117.213915:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d05b7376 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322642240:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (309us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316117.213920:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2550 00000100:00100000:1.0:1761316117.213925:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5177 00000100:00100000:0.0:1761316117.213928:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322642432:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316117.213939:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f937f1e4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322642496:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316117.213946:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5178 00000100:00100000:1.0:1761316117.213962:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322642496:0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761316117.213966:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316117.214006:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000039b44fd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322642624:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316117.214013:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322642048:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316117.214014:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322642368:12345-0@lo:8:kworker/u8:0.0 Request processed in 158us (386us total) trans 0 rc -19/-19 00010000:02020000:0.0:1761316117.214023:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316117.214027:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2549 00000100:00100000:2.0:1761316117.214054:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322642560:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316117.214058:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322642432:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (366us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316117.214068:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2550 00000100:00100000:2.0:1761316117.214073:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322642240:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316117.214078:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316117.214084:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316117.214088:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322642368:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316117.214094:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000021b9e978 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322642176:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316117.214103:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316117.214107:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316117.214112:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322642432:0@lo:8:kworker/u8:0.0 00010000:00010000:2.0:1761316118.236895:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316119.241821:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316119.241874:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316119.241916:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316119.241953:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582848 00000100:00100000:3.0:1761316119.241969:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582784 00000100:00100000:2.0:1761316119.242002:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5179 00000100:00100000:2.0:1761316119.242010:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307582848:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316119.242013:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5180 00000100:00100000:3.0:1761316119.242021:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307582784:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316119.242077:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307582848:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 68us (204us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316119.242084:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5179 00000100:00100000:1.0:1761316119.242114:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582720 00000100:00100000:1.0:1761316119.242172:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1895 00000100:00100000:1.0:1761316119.242177:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307582720:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316119.242381:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307582784:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 360us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316119.242389:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5180 00000100:00100000:1.0:1761316119.243100:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307582720:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 920us (1285us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316119.243109:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1895 00000100:00100000:3.0:1761316119.244936:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316119.245016:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582912 00000100:00100000:2.0:1761316119.245068:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2551 00000100:00100000:2.0:1761316119.245075:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307582912:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761316119.245171:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316119.245255:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307582912:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 179us (324us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316119.245263:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2551 00010000:00010000:1.0:1761316119.259657:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316119.775376:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316119.776178:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322642688:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316119.776252:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316119.776389:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642688 00000100:00100000:3.0:1761316119.776443:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3227 00000100:00100000:3.0:1761316119.776449:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0ca0bde pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322642688:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316119.776483:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316119.776533:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0ca0bde pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322642688:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (281us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316119.776540:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3227 00000100:00100000:1.0:1761316119.776667:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322642688:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316120.284456:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316121.307826:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316122.331511:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316122.331527:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322642752:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316122.331588:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316122.331686:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642752 00000100:00100000:2.0:1761316122.331724:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3228 00000100:00100000:2.0:1761316122.331729:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039391167 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322642752:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316122.331758:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316122.331881:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d940ebd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322642880:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.331908:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316122.331924:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316122.331928:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dce5af8c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322642816:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.331949:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316122.331968:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642816 00000100:00100000:1.0:1761316122.331991:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1761316122.331995:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322642816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.332021:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322642816:12345-0@lo:400:kworker/u8:0.0 Request processed in 26us (72us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316122.332027:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1896 00000100:00100000:1.0:1761316122.332071:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dce5af8c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322642816:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.332107:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322643008:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316122.332132:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316122.332147:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316122.332151:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000df3f2744 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322642944:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.332166:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316122.332186:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643008 00000100:00100000:1.0:1761316122.332656:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1098 00000100:00100000:1.0:1761316122.332663:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322643008:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316122.332688:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316122.333697:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039391167 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322642752:12345-0@lo:41:osp-pre-0-1.0 Request processed in 1962us (2105us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316122.333711:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3228 00000100:00080000:0.0:1761316122.333714:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316122.333721:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316122.333724:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316122.333729:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4364 00000100:00080000:0.0:1761316122.333732:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316122.333736:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316122.333757:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000baba2479 x1846869322643136/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316122.333760:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e6c79737 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322643072:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761316122.333785:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316122.333789:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316122.333790:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316122.333794:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4364 00000100:00080000:0.0:1761316122.333796:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316122.333799:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316122.333808:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316122.333813:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006e647c7d x1846869322643200/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316122.333836:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316122.333844:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322643264:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316122.334063:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642880 00000100:00100000:3.0:1761316122.334097:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322642944 00000100:00100000:0.0:1761316122.334329:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000baba2479 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322643136:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316122.334341:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322643008:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (601us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316122.334365:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1098 00000100:00100000:0.0:1761316122.334390:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316122.334410:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e647c7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322643200:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316122.334433:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316122.334437:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316122.334467:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316122.334483:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643264 00000100:00100000:0.0:1761316122.334485:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643136 00000100:00100000:0.0:1761316122.334513:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643200 00000100:00100000:3.0:1761316122.334532:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5181 00000100:00100000:3.0:1761316122.334540:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007305f3d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322642880:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.334547:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2552 00000100:00100000:0.0:1761316122.334552:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322643136:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316122.334588:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007305f3d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322642880:12345-0@lo:400:kworker/u8:0.0 Request processed in 49us (2680us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316122.334619:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5182 00000100:00100000:1.0:1761316122.334623:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001418520e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322642944:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:0.0:1761316122.334636:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316122.334659:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001418520e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322642944:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (2491us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316122.334666:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5182 00000100:00100000:0.0:1761316122.334690:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322643136:12345-0@lo:8:kworker/u8:0.0 Request processed in 136us (301us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316122.334698:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2552 00000100:00100000:1.0:1761316122.334701:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322643392:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.334715:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2553 00000100:00100000:0.0:1761316122.334720:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322643200:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316122.334733:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316122.334762:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000df3f2744 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322642944:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.334778:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322643008:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316122.334795:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00010000:02020000:0.0:1761316122.334798:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316122.334803:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aa15f0bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322643328:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316122.334834:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316122.334839:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322643200:12345-0@lo:8:kworker/u8:0.0 Request processed in 115us (404us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316122.334846:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2553 00000100:00100000:0.0:1761316122.334862:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643392 00000100:00100000:0.0:1761316122.334884:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1662 00000100:00100000:0.0:1761316122.334887:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cf388c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322643392:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.334943:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cf388c6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322643392:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316122.334944:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643328 00000100:00100000:0.0:1761316122.334951:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1662 00000100:00100000:2.0:1761316122.334965:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5183 00000100:00100000:2.0:1761316122.334970:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b33fd587 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322643264:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761316122.334992:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316122.334997:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316122.335001:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000baba2479 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322643136:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316122.335010:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b33fd587 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322643264:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (1124us total) trans 0 rc 0/0 00000100:00080000:0.0:1761316122.335013:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316122.335017:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316122.335020:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5183 00000100:00100000:0.0:1761316122.335023:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e647c7d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322643200:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316122.335036:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5184 00000100:00100000:2.0:1761316122.335040:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000070eedddd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322643328:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.335065:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322643392:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316122.335070:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000070eedddd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322643328:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (236us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316122.335077:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5184 00000100:00100000:0.0:1761316122.335086:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aa15f0bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322643328:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316122.335112:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322643264:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.335121:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d940ebd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322642880:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316122.335143:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643072 00000100:00100000:2.0:1761316122.335170:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3229 00000100:00100000:2.0:1761316122.335174:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0ce68db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322643072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316122.335178:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322642752:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316122.335224:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0ce68db pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322643072:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (1416us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316122.335230:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3229 00000100:00100000:2.0:1761316122.335258:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e6c79737 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322643072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316122.335413:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5181 00010000:00010000:0.0:1761316123.355996:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316124.366290:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316124.366339:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316124.366418:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316124.366447:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316124.366530:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583168 00000100:00100000:1.0:1761316124.366583:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2554 00000100:00100000:1.0:1761316124.366590:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048964dbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307583168:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761316124.366689:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316124.366746:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048964dbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307583168:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 155us (407us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316124.366754:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2554 00000100:00100000:3.0:1761316124.366836:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307582976 00000100:00100000:3.0:1761316124.366875:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1897 00000100:00100000:3.0:1761316124.366880:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307582976:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316124.366914:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e524c72 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307582976:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (497us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316124.366920:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1897 00000100:00100000:3.0:1761316124.368029:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583040 00000100:00100000:3.0:1761316124.368062:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583104 00000100:00100000:3.0:1761316124.368079:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5185 00000100:00100000:3.0:1761316124.368085:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df276d74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307583040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316124.368136:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df276d74 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307583040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 50us (1853us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316124.368142:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5185 00000100:00100000:3.0:1761316124.368154:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5186 00000100:00100000:3.0:1761316124.368158:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c7dab4a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307583104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316124.368181:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c7dab4a1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307583104:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (1734us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316124.368186:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5186 00010000:00010000:2.0:1761316124.380132:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316124.892673:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316124.892696:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322643456:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316124.892762:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316124.892869:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643456 00000100:00100000:1.0:1761316124.893187:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3230 00000100:00100000:1.0:1761316124.893203:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d940ebd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322643456:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316124.893237:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316124.893298:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d940ebd0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322643456:12345-0@lo:41:osp-pre-1-0.0 Request processed in 95us (537us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316124.893306:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3230 00000100:00100000:2.0:1761316124.893338:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000812c1f62 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322643456:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316125.403777:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316126.427494:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316127.451518:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316127.451524:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316127.451539:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322643520:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316127.451541:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c32f389e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322643648:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316127.451542:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316127.451555:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007de3c26d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322643776:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316127.451564:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316127.451570:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316127.451573:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316127.451579:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4369 00000100:00080000:2.0:1761316127.451582:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316127.451587:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316127.451605:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000028173c34 x1846869322643968/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316127.451617:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451617:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761316127.451623:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1761316127.451628:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316127.451628:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316127.451630:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316127.451634:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4369 00000100:00080000:2.0:1761316127.451636:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:2.0:1761316127.451639:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316127.451649:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000376102c7 x1846869322644032/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316127.451650:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643648 00000100:00100000:0.0:1761316127.451653:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322643584:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316127.451653:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322643712:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.451691:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451691:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316127.451699:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1898 00000100:00100000:1.0:1761316127.451709:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322643648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.451715:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643584 00000100:00100000:0.0:1761316127.451716:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d96cae35 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322643904:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316127.451716:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fabac441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322644160:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.451748:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451748:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316127.451750:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1099 00000100:00100000:2.0:1761316127.451756:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa15f0bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322643584:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316127.451768:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006dae5427 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322643648:12345-0@lo:400:kworker/u8:2.0 Request processed in 59us (154us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316127.451772:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316127.451773:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003e303c6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322644288:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316127.451779:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1898 00002000:00000024:2.0:1761316127.451779:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316127.451781:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322644096:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.451798:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451816:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316127.451829:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa15f0bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322643584:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (139us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316127.451831:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316127.451832:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c32f389e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322643648:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.451843:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1099 00000100:00100000:3.0:1761316127.451851:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643712 00000100:00100000:2.0:1761316127.451872:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644288 00000100:00100000:0.0:1761316127.451876:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322643584:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316127.451883:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643776 00000100:00100000:0.0:1761316127.451897:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761316127.451905:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cacbfc81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322643840:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316127.451908:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644160 00000100:00100000:2.0:1761316127.451909:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1663 00000100:00100000:2.0:1761316127.451916:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e647c7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322644288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316127.451919:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643904 00000100:00100000:0.0:1761316127.451934:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451935:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644096 00000100:00100000:1.0:1761316127.451944:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643520 00000100:00100000:3.0:1761316127.451955:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643840 00000100:00100000:0.0:1761316127.451957:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002a18579d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322644224:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.451958:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e647c7d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322644288:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (160us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316127.451967:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3231 00000100:00100000:2.0:1761316127.451970:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1663 00000100:00100000:1.0:1761316127.451977:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322643776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316127.451977:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5187 00000100:00100000:0.0:1761316127.451984:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316127.451985:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322643712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.451995:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5188 00000100:00100000:2.0:1761316127.452001:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322644160:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452010:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003e303c6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322644288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452036:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322643968:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316127.452042:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322643776:12345-0@lo:400:kworker/u8:2.0 Request processed in 67us (428us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452049:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322644160:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316127.452052:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3231 00000100:00100000:2.0:1761316127.452059:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5188 00000100:00100000:0.0:1761316127.452066:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316127.452078:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3232 00000100:00100000:2.0:1761316127.452082:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5189 00000100:00100000:0.0:1761316127.452087:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000376102c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322644032:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316127.452088:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005af8062f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322643904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452089:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322644096:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452122:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316127.452131:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322644096:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (314us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452137:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5189 00000100:00100000:1.0:1761316127.452151:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005af8062f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322643904:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (403us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452154:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5190 00000100:00100000:2.0:1761316127.452160:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000baba2479 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322643840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452164:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3233 00000100:00100000:1.0:1761316127.452172:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3232 00000100:00100000:2.0:1761316127.452217:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000baba2479 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322643840:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452223:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5190 00000100:00100000:2.0:1761316127.452239:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644224 00000100:00100000:2.0:1761316127.452256:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322643968 00000100:00100000:2.0:1761316127.452268:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644032 00000100:00100000:2.0:1761316127.452277:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1664 00000100:00100000:2.0:1761316127.452280:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040b31d5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322644224:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452317:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040b31d5e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322644224:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (336us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452322:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1664 00000100:00100000:2.0:1761316127.452331:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2555 00000100:00100000:2.0:1761316127.452334:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322643968:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316127.452439:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316127.452464:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322643968:12345-0@lo:8:kworker/u8:2.0 Request processed in 129us (402us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316127.452469:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2555 00000100:00100000:2.0:1761316127.452479:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2556 00000100:00100000:2.0:1761316127.452481:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322644032:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316127.452550:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316127.452569:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322644032:12345-0@lo:8:kworker/u8:2.0 Request processed in 87us (450us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316127.452574:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2556 00000100:00080000:2.0:1761316127.452600:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316127.452603:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316127.452606:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322643968:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761316127.452612:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316127.452614:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316127.452616:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000376102c7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322644032:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452706:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322644096:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452714:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fabac441 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322644160:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452721:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007de3c26d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322643776:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452731:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322643712:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452747:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c74a1dc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322643520:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316127.452775:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316127.452795:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cacbfc81 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322643840:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316127.452808:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002a18579d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322644224:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316127.452819:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c74a1dc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322643520:12345-0@lo:41:osp-pre-0-1.0 Request processed in 647us (990us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316127.452822:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d96cae35 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322643904:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316127.452981:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322643712:12345-0@lo:400:kworker/u8:2.0 Request processed in 995us (1292us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316127.452988:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5187 00000100:00100000:3.0:1761316127.453043:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322643520:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316127.454897:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3233 00010000:00010000:0.0:1761316128.475621:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316129.481879:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316129.481927:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316129.482202:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583232 00000100:00100000:3.0:1761316129.482283:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1899 00000100:00100000:3.0:1761316129.482291:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307583232:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316129.482442:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059a69050 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307583232:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 150us (574us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316129.482451:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1899 00000100:00100000:2.0:1761316129.486982:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316129.487666:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583360 00000100:00100000:2.0:1761316129.487714:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316129.487765:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583296 00000100:00100000:1.0:1761316129.487796:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5191 00000100:00100000:1.0:1761316129.487803:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d96cae35 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307583360:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316129.487896:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d96cae35 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307583360:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 93us (5968us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316129.487904:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5191 00000100:00100000:1.0:1761316129.487939:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5192 00000100:00100000:1.0:1761316129.487943:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002a18579d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307583296:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316129.487969:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002a18579d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307583296:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (1002us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316129.487974:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5192 00000100:00100000:3.0:1761316129.488398:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583424 00000100:00100000:3.0:1761316129.488437:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2557 00000100:00100000:3.0:1761316129.488442:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cacbfc81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307583424:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316129.488898:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316129.488950:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cacbfc81 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307583424:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 506us (1237us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316129.488960:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2557 00010000:00010000:0.0:1761316129.499552:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316130.012325:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322644352:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316130.012442:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316130.012616:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644352 00000100:00100000:0.0:1761316130.012686:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3234 00000100:00100000:0.0:1761316130.012693:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e341f863 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322644352:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316130.012737:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316130.012800:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e341f863 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322644352:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316130.012808:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3234 00000100:00100000:3.0:1761316130.012854:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c0e1e6f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322644352:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316130.524696:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316131.549082:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316132.571718:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1761316132.571745:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322644416:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316132.571834:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316132.572587:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644416 00000100:00100000:1.0:1761316132.572650:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3235 00000100:00100000:1.0:1761316132.572658:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322644416:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316132.572701:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316132.572772:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322644416:12345-0@lo:41:osp-pre-0-1.0 Request processed in 114us (940us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316132.572780:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3235 00000100:00100000:3.0:1761316132.572874:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322644416:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316132.574737:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316132.575459:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322644480:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316132.575539:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316132.577103:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322644544:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316132.577199:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316132.577280:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644544 00000100:00100000:3.0:1761316132.577337:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1900 00000100:00100000:3.0:1761316132.577368:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322644544:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316132.577426:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009fad7035 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322644544:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (228us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316132.577435:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1900 00000100:00100000:3.0:1761316132.577497:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322644544:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316132.577757:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644480 00000100:00100000:1.0:1761316132.577804:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1100 00000100:00100000:1.0:1761316132.577809:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b9872de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322644480:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316132.577839:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:0.0:1761316132.578110:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316132.578117:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316132.578122:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316132.578128:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4375 00000100:00080000:0.0:1761316132.578131:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316132.578134:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316132.578172:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000031d3c62d x1846869322644800/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316132.578190:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316132.578194:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316132.578196:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316132.578199:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4375 00000100:00080000:0.0:1761316132.578200:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:0.0:1761316132.578204:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316132.578216:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000034188510 x1846869322644864/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316132.578264:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b2ab8158 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322644672:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578305:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578326:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645056:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578369:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578388:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1761316132.578393:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322644608:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578414:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578430:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322644992:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578451:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578480:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644672 00000100:00100000:0.0:1761316132.578510:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644608 00000100:00100000:0.0:1761316132.578524:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644992 00000100:00100000:0.0:1761316132.578538:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5193 00000100:00100000:0.0:1761316132.578542:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e6c79737 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322644672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578576:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e6c79737 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322644672:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (271us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.578583:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5193 00000100:00100000:0.0:1761316132.578595:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5194 00000100:00100000:0.0:1761316132.578599:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000df3f2744 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322644608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578621:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000df3f2744 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322644608:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (207us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.578627:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5194 00000100:00100000:0.0:1761316132.578636:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5195 00000100:00100000:0.0:1761316132.578639:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dce5af8c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322644992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578662:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dce5af8c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322644992:12345-0@lo:400:kworker/u8:0.0 Request processed in 23us (212us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.578668:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5195 00000100:00100000:0.0:1761316132.578711:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322644608:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578720:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322644992:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578728:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b2ab8158 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322644672:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578751:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a8e3ffb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322644928:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578773:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578790:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316132.578795:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b9c060f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322644736:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578817:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.578841:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644736 00000100:00100000:0.0:1761316132.578867:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3236 00000100:00100000:0.0:1761316132.578871:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f0cd4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322644736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578901:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f0cd4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322644736:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.578907:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3236 00000100:00100000:0.0:1761316132.578925:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644928 00000100:00100000:0.0:1761316132.578945:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5196 00000100:00100000:0.0:1761316132.578949:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2ab8158 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322644928:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.578976:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2ab8158 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322644928:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (202us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.578982:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5196 00000100:00100000:0.0:1761316132.579016:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b9c060f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322644736:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.579024:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a8e3ffb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322644928:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.579085:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000031d3c62d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322644800:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316132.579115:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.579129:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034188510 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322644864:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316132.579161:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316132.579190:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645056 00000100:00100000:0.0:1761316132.579212:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644800 00000100:00100000:0.0:1761316132.579223:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322644864 00000100:00100000:0.0:1761316132.579234:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1665 00000100:00100000:0.0:1761316132.579238:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645056:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316132.579272:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645056:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (903us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316132.579277:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1665 00000100:00100000:0.0:1761316132.579287:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2558 00000100:00100000:0.0:1761316132.579290:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041cbee93 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322644800:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316132.579531:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2559 00000100:00100000:3.0:1761316132.579536:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a8e3ffb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322644864:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761316132.579617:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316132.579658:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a8e3ffb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322644864:12345-0@lo:8:kworker/u8:0.0 Request processed in 121us (496us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316132.579663:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2559 00000100:00080000:3.0:1761316132.579688:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316132.579691:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316132.579694:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034188510 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322644864:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316132.579761:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645056:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316132.580506:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b9872de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322644480:12345-0@lo:13:osp-pre-1-1.0 Request processed in 2692us (4965us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316132.580518:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1100 00000100:00100000:2.0:1761316132.580619:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322644480:0@lo:13:osp-pre-1-1.0 00010000:02020000:0.0:1761316132.582234:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316132.586121:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041cbee93 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322644800:12345-0@lo:8:kworker/u8:0.0 Request processed in 6816us (6992us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316132.586154:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2558 00000100:00080000:3.0:1761316132.588651:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316132.588665:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316132.588675:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000031d3c62d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322644800:0@lo:8:kworker/u8:0.0 00010000:00010000:1.0:1761316133.595617:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316134.604731:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316134.604790:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316134.604824:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316134.604961:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583488 00000100:00100000:0.0:1761316134.605036:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1901 00000100:00100000:0.0:1761316134.605045:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307583488:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316134.605147:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cde64085 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307583488:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 102us (357us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316134.605155:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1901 00000100:00100000:1.0:1761316134.605934:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583552 00000100:00100000:1.0:1761316134.606001:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583616 00000100:00100000:1.0:1761316134.606028:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5197 00000100:00100000:2.0:1761316134.606037:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5198 00000100:00100000:1.0:1761316134.606041:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007de3c26d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307583552:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316134.606049:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fabac441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307583616:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316134.608857:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fabac441 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307583616:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 2798us (4020us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316134.608858:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007de3c26d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307583552:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 2806us (4125us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316134.608885:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5197 00000100:00100000:2.0:1761316134.608903:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5198 00000100:00100000:2.0:1761316134.609035:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316134.609107:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583680 00000100:00100000:1.0:1761316134.609166:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2560 00000100:00100000:1.0:1761316134.609173:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307583680:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761316134.609281:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316134.609381:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307583680:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 206us (346us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316134.609389:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2560 00010000:00010000:0.0:1761316134.619641:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316135.133149:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316135.134003:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322645120:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316135.134103:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316135.134250:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645120 00000100:00100000:0.0:1761316135.134883:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3237 00000100:00100000:0.0:1761316135.134892:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322645120:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316135.134945:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316135.135023:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322645120:12345-0@lo:41:osp-pre-1-0.0 Request processed in 131us (921us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316135.135031:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3237 00000100:00100000:2.0:1761316135.135269:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322645120:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316135.645021:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316136.671444:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316137.691516:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316137.691963:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316137.691984:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d729081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322645184:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316137.692047:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316137.692160:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645184 00000100:00100000:1.0:1761316137.692211:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1101 00000100:00100000:1.0:1761316137.692216:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322645184:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316137.692244:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316137.692288:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322645184:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316137.692295:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1101 00000100:00100000:1.0:1761316137.692367:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d729081 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322645184:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316137.692553:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316137.692559:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316137.692562:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316137.692568:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4380 00000100:00080000:3.0:1761316137.692571:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761316137.692576:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086e279b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322645312:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316137.692577:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316137.692613:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000000352602 x1846869322645568/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316137.692618:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316137.692634:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316137.692637:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316137.692639:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316137.692640:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00080000:3.0:1761316137.692644:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4380 00000100:00080000:3.0:1761316137.692646:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00100000:2.0:1761316137.692649:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000098d5ea39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322645248:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316137.692651:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316137.692669:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005c22bba0 x1846869322645632/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316137.692684:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.692706:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645504:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.692732:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.692751:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322645696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.692776:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.692800:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645248 00000100:00100000:2.0:1761316137.692827:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1902 00000100:00100000:2.0:1761316137.692831:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322645248:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.692862:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e9fb60d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322645248:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (180us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316137.692880:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1902 00000100:00100000:2.0:1761316137.692911:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645504 00000100:00100000:2.0:1761316137.692932:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3238 00000100:00100000:2.0:1761316137.692935:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028173c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.692963:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028173c34 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645504:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (231us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316137.692968:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3238 00000100:00100000:2.0:1761316137.692992:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000098d5ea39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322645248:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.693003:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645504:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.693019:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a8c16f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322645376:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.693039:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.693050:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000328f1831 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322645760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.693068:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.693081:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:2.0:1761316137.693087:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322645440:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.693114:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.693128:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322645824:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693174:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645376 00000100:00100000:0.0:1761316137.693194:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3239 00000100:00100000:0.0:1761316137.693197:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b99c8f0c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322645376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693226:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b99c8f0c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322645376:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (187us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316137.693232:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3239 00000100:00100000:0.0:1761316137.693278:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645312 00000100:00100000:0.0:1761316137.693301:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645696 00000100:00100000:0.0:1761316137.693313:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645760 00000100:00100000:0.0:1761316137.693321:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645440 00000100:00100000:0.0:1761316137.693330:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5199 00000100:00100000:0.0:1761316137.693333:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000376102c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322645312:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693381:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000376102c7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322645312:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (767us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316137.693388:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5199 00000100:00100000:0.0:1761316137.693397:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5200 00000100:00100000:0.0:1761316137.693400:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd918d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322645696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693421:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd918d21 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322645696:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (645us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316137.693425:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5200 00000100:00100000:0.0:1761316137.693433:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5201 00000100:00100000:0.0:1761316137.693436:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000048964dbc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322645760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693454:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000048964dbc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322645760:12345-0@lo:400:kworker/u8:0.0 Request processed in 17us (385us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316137.693459:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5201 00000100:00100000:0.0:1761316137.693468:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5202 00000100:00100000:0.0:1761316137.693470:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cacbfc81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322645440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693490:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cacbfc81 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322645440:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (375us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316137.693494:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5202 00000100:00100000:0.0:1761316137.693529:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322645568:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693556:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316137.693570:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c22bba0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322645632:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693590:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316137.693634:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322645696:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693644:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086e279b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322645312:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316137.693670:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645568 00000100:00100000:0.0:1761316137.693686:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645632 00000100:00100000:0.0:1761316137.693699:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2561 00000100:00100000:0.0:1761316137.693702:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322645568:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761316137.693789:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316137.693821:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322645568:12345-0@lo:8:kworker/u8:0.0 Request processed in 117us (265us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316137.693827:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2561 00000100:00100000:0.0:1761316137.693838:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2562 00000100:00100000:0.0:1761316137.693840:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322645632:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:0.0:1761316137.693914:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316137.693938:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322645632:12345-0@lo:8:kworker/u8:0.0 Request processed in 96us (347us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316137.693943:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2562 00000100:00080000:0.0:1761316137.693966:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316137.693970:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316137.693974:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322645568:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761316137.693983:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316137.693986:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316137.693988:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c22bba0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322645632:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316137.694295:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.694329:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a8c16f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322645376:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.694338:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000328f1831 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322645760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316137.694369:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322645440:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316137.696046:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645824 00000100:00100000:2.0:1761316137.696909:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322645888:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316137.696964:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316137.697013:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645888 00000100:00100000:2.0:1761316137.697044:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3240 00000100:00100000:2.0:1761316137.697050:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c32f389e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322645888:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316137.697078:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316137.697126:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c32f389e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322645888:12345-0@lo:41:osp-pre-0-1.0 Request processed in 76us (162us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316137.697133:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3240 00000100:00100000:2.0:1761316137.697173:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322645888:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316137.700756:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1666 00000100:00100000:3.0:1761316137.700768:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e303c6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322645824:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316137.702984:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e303c6b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322645824:12345-0@lo:400:kworker/u8:0.0 Request processed in 2211us (8683us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316137.702999:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1666 00000100:00100000:0.0:1761316137.703444:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322645824:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316138.715504:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316139.721785:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316139.722464:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316139.722599:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583744 00000100:00100000:2.0:1761316139.722656:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1903 00000100:00100000:2.0:1761316139.722662:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307583744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316139.722727:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f0fb384c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307583744:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (950us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316139.722734:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1903 00000100:00100000:1.0:1761316139.727595:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583872 00000100:00100000:3.0:1761316139.727800:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316139.727952:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583808 00000100:00100000:1.0:1761316139.727977:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5203 00000100:00100000:1.0:1761316139.727983:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3ecfafc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307583872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316139.728053:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3ecfafc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307583872:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (5591us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316139.728061:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5203 00000100:00100000:1.0:1761316139.728078:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5204 00000100:00100000:1.0:1761316139.728082:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069f2cc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307583808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316139.728118:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069f2cc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307583808:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (323us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316139.728124:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5204 00000100:00100000:3.0:1761316139.728300:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316139.731013:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307583936 00000100:00100000:1.0:1761316139.731071:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2563 00000100:00100000:1.0:1761316139.731077:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b347652f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307583936:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316139.731181:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316139.731258:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b347652f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307583936:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 180us (2958us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316139.731268:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2563 00010000:00010000:1.0:1761316139.741134:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316140.251548:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645952:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316140.251621:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316140.255845:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322645952 00000100:00100000:3.0:1761316140.255907:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3241 00000100:00100000:3.0:1761316140.255913:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645952:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316140.255949:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316140.256017:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322645952:12345-0@lo:41:osp-pre-1-0.0 Request processed in 103us (4397us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316140.256024:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3241 00000100:00100000:2.0:1761316140.257042:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322645952:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316140.763710:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316141.787771:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316142.811969:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316142.811988:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322646080:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316142.811996:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316142.812002:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316142.812004:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316142.812010:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4385 00000100:00080000:1.0:1761316142.812013:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316142.812018:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316142.812040:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006cf388c6 x1846869322646336/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316142.812054:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316142.812057:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316142.812061:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316142.812063:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316142.812076:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4385 00000100:00080000:1.0:1761316142.812078:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:1.0:1761316142.812081:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316142.812093:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000168f6c92 x1846869322646400/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316142.812123:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322646464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316142.812150:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316142.812158:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006cf388c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322646336:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316142.812170:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [2->3] 00000100:00100000:0.0:1761316142.812178:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1f6a7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322646016:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812198:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316142.812215:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316142.812222:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322646400:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316142.812233:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4365f07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322646272:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812249:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316142.812265:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316142.812284:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322646656:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812293:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646336 00000100:00100000:0.0:1761316142.812302:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316142.812326:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646080 00000100:00100000:1.0:1761316142.812427:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646400 00000100:00100000:1.0:1761316142.812444:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646656 00000100:00100000:1.0:1761316142.812466:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1667 00000100:00100000:1.0:1761316142.812471:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322646656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812518:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322646656:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (216us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316142.812525:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1667 00000100:00100000:1.0:1761316142.812543:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2564 00000100:00100000:1.0:1761316142.812548:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322646336:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316142.812569:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646144:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.812602:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316142.812617:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322646528:0@lo:400:kworker/u8:2.0 00010000:02020000:1.0:1761316142.812632:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316142.812640:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316142.812668:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761316142.812674:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322646336:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (476us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316142.812676:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322646208:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812688:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2564 00000100:00100000:1.0:1761316142.812701:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2565 00000100:00100000:1.0:1761316142.812705:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322646400:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316142.812710:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316142.812727:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646592:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.812747:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316142.812770:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646464 00010000:02020000:1.0:1761316142.812781:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316142.812804:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322646400:12345-0@lo:8:kworker/u8:2.0 Request processed in 98us (556us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316142.812809:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2565 00000100:00100000:1.0:1761316142.812819:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646592 00000100:00100000:1.0:1761316142.812833:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1668 00000100:00100000:1.0:1761316142.812836:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322646592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812870:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322646592:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (122us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316142.812876:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1668 00000100:00100000:3.0:1761316142.812878:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646272 00000100:00080000:1.0:1761316142.812909:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316142.812913:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316142.812913:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646144 00000100:00100000:1.0:1761316142.812919:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006cf388c6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322646336:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316142.812927:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316142.812930:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316142.812935:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000168f6c92 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322646400:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316142.812936:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3242 00000100:00100000:3.0:1761316142.812944:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322646272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.812960:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322646720:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316142.812988:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322646272:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (724us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316142.812996:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316142.813001:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3242 00000100:00100000:3.0:1761316142.813019:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3243 00000100:00100000:3.0:1761316142.813024:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322646144:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.813028:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4365f07 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322646272:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.813039:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322646656:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316142.813062:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322646144:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (459us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316142.813079:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3243 00000100:00100000:1.0:1761316142.813099:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646016 00000100:00100000:3.0:1761316142.813124:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646528 00000100:00100000:1.0:1761316142.813127:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1904 00000100:00100000:1.0:1761316142.813132:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322646016:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.813162:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004098f9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322646016:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (949us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316142.813167:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1904 00000100:00100000:1.0:1761316142.813193:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1f6a7de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322646016:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316142.813224:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646720 00000100:00100000:1.0:1761316142.813244:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1102 00000100:00100000:1.0:1761316142.813247:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322646720:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316142.813808:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646208 00000100:00100000:2.0:1761316142.813821:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5205 00000100:00100000:2.0:1761316142.813829:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006e647c7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322646080:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316142.813829:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5206 00000100:00100000:3.0:1761316142.813837:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040b31d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322646464:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.813878:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006e647c7d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322646080:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (1827us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316142.813886:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040b31d5e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322646464:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (1735us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316142.813893:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5205 00000100:00100000:3.0:1761316142.813899:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5206 00000100:00100000:2.0:1761316142.813914:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5207 00000100:00100000:2.0:1761316142.813920:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322646528:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.813949:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000960eacc4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322646528:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (1311us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316142.813955:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5207 00000100:00100000:2.0:1761316142.814008:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646592:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.814016:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000077948496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646144:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.814024:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322646528:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316142.814271:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5208 00000100:00100000:0.0:1761316142.814275:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322646208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316142.814306:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1edac59 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322646208:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (1599us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316142.814312:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5208 00000100:00100000:0.0:1761316142.814377:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322646464:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316142.814386:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322646080:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.814682:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322646208:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316142.815153:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646784:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316142.815192:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316142.815894:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646784 00000100:00100000:3.0:1761316142.815947:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3244 00000100:00100000:3.0:1761316142.815954:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322646784:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316142.815984:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316142.816022:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322646784:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (832us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316142.816028:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3244 00002000:00000024:1.0:1761316142.816105:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316142.816250:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322646784:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316142.817458:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322646720:12345-0@lo:13:osp-pre-1-1.0 Request processed in 4206us (4462us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316142.817470:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1102 00000100:00100000:0.0:1761316142.817818:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322646720:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316143.835692:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316144.839976:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316144.840027:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316144.840098:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316144.840123:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316144.840159:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584064 00000100:00100000:3.0:1761316144.840203:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584128 00000100:00100000:2.0:1761316144.840223:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5209 00000100:00100000:3.0:1761316144.840231:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5210 00000100:00100000:2.0:1761316144.840235:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a51ecd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307584064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316144.840243:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000596f0dd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316144.840396:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584000 00000100:00100000:0.0:1761316144.840430:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1905 00000100:00100000:0.0:1761316144.840434:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307584000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316144.840917:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584192 00000100:00100000:1.0:1761316144.840944:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2566 00000100:00100000:1.0:1761316144.840948:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307584192:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316144.841037:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316144.843078:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000043bc7260 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307584000:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 2632us (2971us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316144.843091:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1905 00000100:00100000:3.0:1761316144.843641:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000596f0dd4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584128:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 3398us (3515us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316144.843651:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5210 00000100:00100000:2.0:1761316144.843846:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a51ecd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307584064:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 3613us (3877us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316144.843854:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5209 00000100:00100000:1.0:1761316144.844125:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307584192:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 3174us (4097us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316144.844134:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2566 00010000:00010000:0.0:1761316144.861692:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316145.372464:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322646848:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316145.372538:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316145.372597:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646848 00000100:00100000:2.0:1761316145.372651:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3245 00000100:00100000:2.0:1761316145.372658:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322646848:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316145.372688:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316145.372742:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322646848:12345-0@lo:41:osp-pre-1-0.0 Request processed in 83us (204us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316145.372749:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3245 00000100:00100000:0.0:1761316145.377339:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322646848:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316145.884130:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316146.907505:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316147.932275:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b9c060f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322646912:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316147.932387:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316147.932499:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646912 00000100:00100000:1.0:1761316147.932544:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3246 00000100:00100000:1.0:1761316147.932551:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322646912:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316147.932572:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316147.932581:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000627dfbad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322646976:0@lo:13:osp-pre-1-1.0 00000004:00000024:1.0:1761316147.932582:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316147.932617:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316147.932631:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316147.932636:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316147.932640:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316147.932646:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4390 00000100:00080000:3.0:1761316147.932648:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316147.932653:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316147.932661:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b9c060f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322646912:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316147.932672:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ea95e939 x1846869322647296/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316147.932681:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322647168:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316147.932689:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316147.932694:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316147.932696:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316147.932700:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4390 00000100:00080000:3.0:1761316147.932702:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 21s 00000100:00080000:3.0:1761316147.932705:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316147.932712:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316147.932719:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000033e2b8a8 x1846869322647360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316147.932732:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316147.932739:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322647232:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316147.932760:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316147.932784:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647232 00000100:00100000:1.0:1761316147.932785:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322646912:12345-0@lo:41:osp-pre-0-1.0 Request processed in 232us (396us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316147.932794:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3246 00000100:00100000:2.0:1761316147.932807:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3247 00000100:00100000:2.0:1761316147.932814:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322647232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.932823:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322647296:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316147.932855:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322647232:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (94us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316147.932864:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3247 00000100:00100000:1.0:1761316147.932867:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316147.932890:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322646976 00000100:00100000:1.0:1761316147.932891:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322647360:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316147.932917:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316147.932920:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1103 00000100:00100000:2.0:1761316147.932925:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6470a0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322646976:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316147.932946:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316147.932955:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647296 00000100:00100000:1.0:1761316147.932979:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647360 00000100:00100000:2.0:1761316147.932985:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6470a0b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322646976:12345-0@lo:13:osp-pre-1-1.0 Request processed in 59us (370us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316147.932995:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322647104:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316147.932998:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1103 00000100:00100000:1.0:1761316147.933003:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2567 00000100:00100000:1.0:1761316147.933008:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322647296:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316147.933118:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316147.933156:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322647296:12345-0@lo:8:kworker/u8:0.0 Request processed in 146us (293us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316147.933162:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2567 00000100:00100000:3.0:1761316147.933169:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316147.933182:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2568 00000100:00100000:1.0:1761316147.933188:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322647360:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933199:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322647488:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933230:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316147.933251:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761316147.933272:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647168 00010000:02020000:1.0:1761316147.933273:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316147.933300:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647104 00000100:00100000:1.0:1761316147.933304:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322647360:12345-0@lo:8:kworker/u8:0.0 Request processed in 117us (387us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316147.933312:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2568 00000100:00100000:2.0:1761316147.933320:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647488 00000100:00100000:2.0:1761316147.933365:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5211 00000100:00080000:1.0:1761316147.933368:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1761316147.933371:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322647168:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316147.933373:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316147.933378:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322647296:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761316147.933387:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316147.933389:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316147.933404:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322647360:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933446:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316147.933455:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322647552:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316147.933461:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5212 00000100:00100000:0.0:1761316147.933467:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000328f1831 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322647104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933482:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316147.933508:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000328f1831 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322647104:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (476us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316147.933514:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5212 00010000:00010000:1.0:1761316147.933520:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316147.933527:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5213 00000100:00100000:0.0:1761316147.933531:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a8c16f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322647488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933538:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647552 00000100:00100000:0.0:1761316147.933553:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a8c16f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322647488:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (323us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316147.933560:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5213 00000100:00100000:1.0:1761316147.933571:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1669 00000100:00100000:1.0:1761316147.933577:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322647552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933588:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322647040:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316147.933625:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322647232:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933626:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322647552:12345-0@lo:400:kworker/u8:0.0 Request processed in 46us (143us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316147.933635:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316147.933640:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1669 00000100:00100000:0.0:1761316147.933648:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000627dfbad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322646976:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316147.933660:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322647424:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933671:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647040 00000100:00100000:1.0:1761316147.933701:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1906 00000100:00100000:3.0:1761316147.933702:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316147.933709:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322647040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933785:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322647104:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933790:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002f5736db pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322647040:12345-0@lo:400:kworker/u8:0.0 Request processed in 80us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316147.933798:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322647488:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316147.933800:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1906 00000100:00100000:3.0:1761316147.933822:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322647040:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933851:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647424 00000100:00100000:3.0:1761316147.933879:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5214 00000100:00100000:3.0:1761316147.933884:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c22bba0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322647424:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316147.933919:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322647552:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316147.933920:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c22bba0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322647424:12345-0@lo:400:kworker/u8:0.0 Request processed in 34us (220us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316147.933931:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5214 00000100:00100000:1.0:1761316147.933937:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322647424:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316147.934368:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f595e60f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322647168:12345-0@lo:400:kworker/u8:0.0 Request processed in 997us (1657us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316147.934375:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5211 00000100:00100000:0.0:1761316147.934379:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322647168:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316148.955635:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316149.960821:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316149.961064:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584256 00000100:00100000:0.0:1761316149.961076:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316149.961121:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316149.961138:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584448 00000100:00100000:2.0:1761316149.961175:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2569 00000100:00100000:0.0:1761316149.961180:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316149.961184:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004acac90c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307584448:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761316149.961232:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584384 00000100:00100000:0.0:1761316149.961266:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584320 00010000:02020000:2.0:1761316149.961278:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316149.961284:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5215 00000100:00100000:0.0:1761316149.961289:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000039b44fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307584384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316149.961369:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004acac90c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307584448:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 158us (265us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316149.961379:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2569 00000100:00100000:0.0:1761316149.961383:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000039b44fd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307584384:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 51us (222us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316149.961392:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5215 00000100:00100000:0.0:1761316149.961412:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5216 00000100:00100000:0.0:1761316149.961417:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa15f0bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307584320:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316149.961444:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa15f0bc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307584320:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 27us (273us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316149.961449:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5216 00000100:00100000:1.0:1761316149.961665:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1907 00000100:00100000:1.0:1761316149.961670:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307584256:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316149.961716:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea0688c5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307584256:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 45us (903us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316149.961722:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1907 00010000:00010000:0.0:1761316149.979579:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316150.491911:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322647616:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316150.491981:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316150.492499:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647616 00000100:00100000:1.0:1761316150.492562:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3248 00000100:00100000:1.0:1761316150.492568:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322647616:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316150.492600:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316150.492655:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322647616:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (676us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316150.492663:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3248 00000100:00100000:0.0:1761316150.492713:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322647616:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316151.004055:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316152.028802:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316153.051517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316153.051537:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322647680:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316153.051591:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322647808:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.051631:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316153.051727:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322647744:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316153.051766:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316153.051803:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647808 00000100:00100000:2.0:1761316153.051845:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5217 00000100:00100000:2.0:1761316153.051850:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b9c060f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322647808:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.052500:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316153.052548:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647744 00000100:00100000:1.0:1761316153.052580:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1908 00000100:00100000:1.0:1761316153.052584:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322647744:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.052629:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000972d38bd pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322647744:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (863us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316153.052635:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1908 00000100:00100000:1.0:1761316153.052678:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086e279b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322647872:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.052703:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316153.052734:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647872 00000100:00100000:2.0:1761316153.052760:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b9c060f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322647808:12345-0@lo:400:kworker/u8:2.0 Request processed in 907us (1127us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316153.052763:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3249 00000100:00100000:1.0:1761316153.052769:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322647872:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316153.052774:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5217 00000100:00100000:1.0:1761316153.052814:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322647872:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (109us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316153.052821:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322647808:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.052822:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3249 00000100:00100000:2.0:1761316153.052844:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322647744:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.052867:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a3c5b3a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322647936:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316153.052871:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086e279b6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322647872:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.052898:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316153.052898:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647680 00000100:00100000:1.0:1761316153.052919:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322647936 00000100:00100000:2.0:1761316153.052933:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1104 00000100:00100000:2.0:1761316153.052939:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322647680:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316153.052950:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3250 00000100:00100000:1.0:1761316153.052956:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322647936:12345-0@lo:41:osp-pre-0-1.0 00002000:00000024:2.0:1761316153.052963:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000004:00000024:1.0:1761316153.052979:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316153.053022:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abeba833 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322647680:12345-0@lo:13:osp-pre-1-1.0 Request processed in 82us (523us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316153.053030:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1104 00000100:00100000:1.0:1761316153.053033:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322647936:12345-0@lo:41:osp-pre-0-1.0 Request processed in 76us (136us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316153.053042:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3250 00000100:00100000:0.0:1761316153.053055:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a3c5b3a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322647936:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316153.053072:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322647680:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316153.053258:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322648000:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316153.053283:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316153.053334:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648000 00000100:00100000:0.0:1761316153.053381:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5218 00000100:00100000:0.0:1761316153.053385:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322648000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.055216:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b46f99e6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322648000:12345-0@lo:400:kworker/u8:2.0 Request processed in 1824us (1927us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316153.055230:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5218 00000100:00100000:2.0:1761316153.055361:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322648000:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316153.055630:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316153.055635:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316153.055639:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316153.055643:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4395 00000100:00100000:2.0:1761316153.055652:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322648064:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316153.055655:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 6s 00000100:00080000:3.0:1761316153.055659:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316153.055678:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000098d5ea39 x1846869322648128/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316153.055693:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316153.055694:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316153.055695:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316153.055697:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316153.055700:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4395 00000100:00080000:3.0:1761316153.055701:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 6s 00000100:00080000:3.0:1761316153.055704:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316153.055714:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000978c22f3 x1846869322648192/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316153.055741:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322648256:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316153.055767:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316153.055770:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000098d5ea39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322648128:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055796:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316153.055801:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322648384:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055810:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322648192:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055836:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648128 00000100:00100000:2.0:1761316153.055850:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316153.055858:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648384 00000100:00100000:2.0:1761316153.055868:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316153.055872:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322648320:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055880:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1670 00000100:00100000:3.0:1761316153.055884:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322648384:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316153.055891:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316153.055919:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2570 00000100:00100000:2.0:1761316153.055923:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086e279b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322648128:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055923:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322648384:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (74us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316153.055929:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1670 00000100:00100000:3.0:1761316153.055952:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db8d9d28 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322648384:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316153.055971:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316153.055990:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648192 00000100:00100000:3.0:1761316153.056014:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2571 00000100:00100000:3.0:1761316153.056018:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db8d9d28 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322648192:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316153.056026:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316153.056056:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086e279b6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322648128:12345-0@lo:8:kworker/u8:2.0 Request processed in 133us (261us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316153.056063:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2570 00000100:00080000:2.0:1761316153.056092:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316153.056095:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316153.056098:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000098d5ea39 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322648128:0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316153.056098:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316153.056146:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db8d9d28 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322648192:12345-0@lo:8:kworker/u8:2.0 Request processed in 126us (173us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316153.056151:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2571 00000100:00080000:2.0:1761316153.056260:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316153.056263:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316153.056266:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322648192:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316153.056873:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648256 00000100:00100000:0.0:1761316153.056910:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648320 00000100:00100000:0.0:1761316153.056937:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5219 00000100:00100000:0.0:1761316153.056941:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322648256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.056983:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000f69efca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322648256:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (1216us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316153.056989:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5219 00000100:00100000:0.0:1761316153.057010:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5220 00000100:00100000:0.0:1761316153.057014:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322648320:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316153.057042:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322648320:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (1150us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316153.057047:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5220 00000100:00100000:2.0:1761316153.057176:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322648256:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316153.059828:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c731958 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322648320:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.060231:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648064 00000100:00100000:1.0:1761316153.060292:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3251 00000100:00100000:1.0:1761316153.060298:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322648064:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316153.060418:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322648064:12345-0@lo:400:kworker/u8:2.0 Request processed in 119us (4722us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316153.060424:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3251 00000100:00100000:2.0:1761316153.064382:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322648064:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761316154.075540:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316155.081251:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316155.081289:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316155.081321:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316155.081340:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316155.081399:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584640 00000100:00100000:3.0:1761316155.081439:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584576 00000100:00100000:3.0:1761316155.081455:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5221 00000100:00100000:3.0:1761316155.081462:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000b653f5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316155.081527:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000b653f5d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584640:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (206us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316155.081535:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5221 00000100:00100000:3.0:1761316155.081551:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5222 00000100:00100000:3.0:1761316155.081555:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000352602 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307584576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316155.081579:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000352602 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307584576:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (238us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316155.081585:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5222 00000100:00100000:2.0:1761316155.081788:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584704 00000100:00100000:2.0:1761316155.081821:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2572 00000100:00100000:2.0:1761316155.081824:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c731958 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307584704:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316155.081912:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316155.081955:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c731958 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307584704:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 129us (666us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316155.081962:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2572 00000100:00100000:1.0:1761316155.082040:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584512 00000100:00100000:1.0:1761316155.082078:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1909 00000100:00100000:1.0:1761316155.082082:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307584512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316155.082110:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021517f69 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307584512:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 27us (866us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316155.082115:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1909 00010000:00010000:2.0:1761316155.106179:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316155.611821:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316155.611842:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322648448:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316155.611912:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316155.612008:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648448 00000100:00100000:3.0:1761316155.612061:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3252 00000100:00100000:3.0:1761316155.612066:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000031d3c62d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322648448:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316155.612098:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316155.612147:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000031d3c62d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322648448:12345-0@lo:41:osp-pre-1-0.0 Request processed in 81us (236us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316155.612154:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3252 00000100:00100000:0.0:1761316155.612242:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322648448:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316156.124802:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316157.149715:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761316158.171595:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316158.171601:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316158.171604:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316158.171611:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4400 00000100:00080000:3.0:1761316158.171615:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 11s 00000100:00080000:3.0:1761316158.171620:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316158.171640:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000033e2b8a8 x1846869322648832/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316158.171653:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316158.171655:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316158.171657:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316158.171660:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4400 00000100:00080000:3.0:1761316158.171661:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 11s 00000100:00080000:3.0:1761316158.171663:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316158.171669:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322648576:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.171675:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ea95e939 x1846869322648896/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316158.171733:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322648640:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316158.171749:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.171850:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000627dfbad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322648704:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.171933:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316158.171957:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322649024:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.171990:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316158.172006:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316158.172012:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322648960:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.172033:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316158.172079:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648640 00000100:00100000:3.0:1761316158.172117:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3253 00000100:00100000:3.0:1761316158.172122:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000098d5ea39 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322648640:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316158.172156:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648576 00000100:00100000:3.0:1761316158.172179:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000098d5ea39 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322648640:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (246us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316158.172188:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3253 00000100:00100000:1.0:1761316158.172195:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649024 00000100:00100000:1.0:1761316158.172217:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648960 00000100:00100000:0.0:1761316158.172238:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5223 00000100:00100000:0.0:1761316158.172246:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322648576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316158.172282:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000021b9e978 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322648576:12345-0@lo:400:kworker/u8:2.0 Request processed in 37us (536us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316158.172305:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5224 00000100:00100000:1.0:1761316158.172311:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008c8247c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322649024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172390:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316158.172405:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008c8247c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322649024:12345-0@lo:400:kworker/u8:2.0 Request processed in 92us (413us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.172408:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322649088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172434:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172455:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [3->2] 00000100:00100000:2.0:1761316158.172459:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322648512:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172481:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172499:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322648768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172519:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172531:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322649152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172557:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172580:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322648832:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172601:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172613:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322648896:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172632:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.172665:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649088 00000100:00100000:2.0:1761316158.172692:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649152 00000100:00100000:2.0:1761316158.172709:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648832 00000100:00100000:2.0:1761316158.172717:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648896 00000100:00100000:2.0:1761316158.172726:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1671 00000100:00100000:2.0:1761316158.172730:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322649088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172773:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322649088:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (339us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.172780:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1671 00000100:00100000:2.0:1761316158.172791:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1672 00000100:00100000:2.0:1761316158.172795:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322649152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.172820:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322649152:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (268us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.172828:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1672 00000100:00100000:2.0:1761316158.172838:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2573 00000100:00100000:2.0:1761316158.172841:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322648832:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316158.172932:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316158.172973:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322648832:12345-0@lo:8:kworker/u8:2.0 Request processed in 130us (371us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316158.172979:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2573 00000100:00100000:2.0:1761316158.172989:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2574 00000100:00100000:2.0:1761316158.172992:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322648896:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316158.173064:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316158.173082:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322648896:12345-0@lo:8:kworker/u8:2.0 Request processed in 89us (450us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316158.173086:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2574 00000100:00100000:2.0:1761316158.173115:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648512 00000100:00100000:2.0:1761316158.173134:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1910 00000100:00100000:2.0:1761316158.173138:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322648512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173160:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd8f5cd6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322648512:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (679us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.173165:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1910 00000100:00080000:2.0:1761316158.173587:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316158.173591:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316158.173596:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e2b8a8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322648832:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761316158.173608:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316158.173611:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316158.173613:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ea95e939 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322648896:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173653:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648768 00000100:00100000:2.0:1761316158.173680:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3254 00000100:00100000:2.0:1761316158.173683:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322648768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173715:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322648768:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (1196us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.173721:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3254 00000100:00100000:2.0:1761316158.173746:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322648512:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173756:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322648768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000978c22f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322649152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173774:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000842edb0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322649088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173810:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649280:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316158.173834:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.173858:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322648640:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173869:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f54c1b9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322649024:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.173881:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316158.173885:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649216:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316158.173905:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316158.173928:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649280 00000100:00100000:2.0:1761316158.173953:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1105 00000100:00100000:2.0:1761316158.173956:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000842edb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322649280:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316158.173988:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316158.174022:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000842edb0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322649280:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (187us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.174029:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1105 00000100:00100000:2.0:1761316158.174045:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649216 00000100:00100000:2.0:1761316158.174064:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3255 00000100:00100000:2.0:1761316158.174067:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322649216:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316158.174088:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316158.174114:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322649216:12345-0@lo:41:osp-pre-0-1.0 Request processed in 46us (209us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.174119:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3255 00000100:00100000:2.0:1761316158.174147:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649216:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316158.174158:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649280:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316158.174204:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322648704 00000100:00100000:3.0:1761316158.174267:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5225 00000100:00100000:0.0:1761316158.174302:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5223 00000100:00100000:3.0:1761316158.174325:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322648960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.174388:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c145649f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322648960:12345-0@lo:400:kworker/u8:2.0 Request processed in 63us (2354us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316158.174395:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5225 00000100:00100000:3.0:1761316158.174424:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000590ef1c4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322648960:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316158.174482:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b596d59b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322648576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316158.174563:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5224 00010000:00010000:3.0:1761316158.174726:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316158.175782:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5226 00000100:00100000:2.0:1761316158.175791:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322648704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316158.175988:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322648704:12345-0@lo:400:kworker/u8:2.0 Request processed in 198us (3599us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316158.175996:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5226 00000100:00100000:3.0:1761316158.176256:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000627dfbad pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322648704:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316159.195580:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316160.203357:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316160.203540:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584768 00000100:00100000:0.0:1761316160.203724:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1911 00000100:00100000:0.0:1761316160.203731:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307584768:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316160.203800:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000149d2c35 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307584768:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 69us (1479us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316160.203807:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1911 00000100:00100000:0.0:1761316160.205383:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316160.206189:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316160.206258:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584896 00000100:00100000:3.0:1761316160.206311:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584960 00000100:00100000:2.0:1761316160.206322:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5227 00000100:00100000:2.0:1761316160.206331:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034188510 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316160.206418:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2575 00000100:00100000:3.0:1761316160.206427:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b867a88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307584960:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761316160.206443:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034188510 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307584896:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 111us (1062us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316160.206457:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5227 00010000:02020000:3.0:1761316160.206533:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316160.206613:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b867a88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307584960:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 186us (426us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316160.206621:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2575 00000100:00100000:0.0:1761316160.210955:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316160.212508:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307584832 00000100:00100000:1.0:1761316160.212577:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5228 00000100:00100000:1.0:1761316160.212586:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307584832:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316160.212679:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f89f3b4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307584832:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 94us (1743us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316160.212686:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5228 00010000:00010000:2.0:1761316160.219729:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316160.731620:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316160.731827:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322649344:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316160.731895:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316160.732000:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649344 00000100:00100000:2.0:1761316160.732056:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3256 00000100:00100000:2.0:1761316160.732063:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322649344:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316160.732097:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316160.732154:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322649344:12345-0@lo:41:osp-pre-1-0.0 Request processed in 91us (260us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316160.732162:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3256 00000100:00100000:1.0:1761316160.733269:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322649344:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316161.243541:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316162.267536:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316163.291515:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316163.291515:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316163.291526:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316163.291537:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322649472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316163.291540:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649408:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316163.291605:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.291610:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.291672:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649472 00000100:00100000:3.0:1761316163.291718:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1106 00000100:00100000:3.0:1761316163.291726:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322649472:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316163.291742:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00002000:00000024:3.0:1761316163.291754:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316163.291756:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322649600:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316163.291761:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316163.291767:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316163.291771:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316163.291776:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4405 00000100:00080000:1.0:1761316163.291779:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 16s 00000100:00080000:1.0:1761316163.291783:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316163.291798:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.291801:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e10a0c4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322649472:12345-0@lo:13:osp-pre-1-1.0 Request processed in 74us (191us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316163.291802:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ba4634d0 x1846869322649792/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316163.291810:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1106 00000100:00080000:1.0:1761316163.291819:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316163.291823:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761316163.291824:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322649664:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316163.291825:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316163.291831:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4405 00000100:00080000:1.0:1761316163.291833:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 16s 00000100:00080000:1.0:1761316163.291836:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316163.291849:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b893325d x1846869322649856/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316163.291856:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.291873:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322649472:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316163.291880:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316163.291890:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322650048:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316163.291899:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba4634d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322649792:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316163.291909:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649600 00000100:00100000:2.0:1761316163.291917:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316163.291951:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322649536:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316163.291960:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649664 00000100:00100000:2.0:1761316163.291967:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322649984:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316163.291982:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.291982:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5229 00000100:00100000:3.0:1761316163.291987:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a9e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322649600:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.292003:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316163.292004:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322649920:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316163.292029:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.292035:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a9e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322649600:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (240us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316163.292038:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322649600:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316163.292048:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316163.292050:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5229 00000100:00100000:1.0:1761316163.292054:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322649728:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316163.292066:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649984 00000100:00100000:1.0:1761316163.292073:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316163.292076:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649920 00000100:00100000:3.0:1761316163.292086:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5230 00000100:00100000:3.0:1761316163.292089:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099aed9bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322649664:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316163.292098:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5231 00000100:00100000:1.0:1761316163.292101:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033e2b8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322649984:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316163.292155:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650048 00000100:00100000:0.0:1761316163.292187:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1673 00000100:00100000:0.0:1761316163.292191:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea95e939 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322650048:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.292316:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1761316163.292322:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077948496 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322649920:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316163.292327:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099aed9bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322649664:12345-0@lo:400:kworker/u8:0.0 Request processed in 234us (473us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316163.292336:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5230 00000100:00100000:2.0:1761316163.292382:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077948496 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322649920:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (353us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316163.292390:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5232 00000100:00100000:2.0:1761316163.292426:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649408 00000100:00100000:2.0:1761316163.292445:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649728 00000100:00100000:2.0:1761316163.292458:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3257 00000100:00100000:2.0:1761316163.292461:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322649408:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316163.292480:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316163.292503:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322649408:12345-0@lo:41:osp-pre-0-1.0 Request processed in 41us (899us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316163.292508:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3257 00000100:00100000:2.0:1761316163.292518:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3258 00000100:00100000:2.0:1761316163.292521:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1f6a7de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322649728:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.292540:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1f6a7de pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322649728:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (467us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316163.292545:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3258 00000100:00100000:2.0:1761316163.292563:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649536 00000100:00100000:2.0:1761316163.292583:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1912 00000100:00100000:2.0:1761316163.292586:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322649536:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.292609:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013c10909 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322649536:12345-0@lo:400:kworker/u8:0.0 Request processed in 22us (628us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316163.292614:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1912 00000100:00100000:2.0:1761316163.292679:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316163.292693:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322649856:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316163.292714:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316163.292735:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649792 00000100:00100000:2.0:1761316163.292748:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322649856 00000100:00100000:2.0:1761316163.292758:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2576 00000100:00100000:2.0:1761316163.292762:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322649792:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316163.292846:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316163.292873:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322649792:12345-0@lo:8:kworker/u8:0.0 Request processed in 110us (195us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316163.292879:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2576 00000100:00100000:2.0:1761316163.292888:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2577 00000100:00100000:2.0:1761316163.292891:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322649856:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316163.292979:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316163.292998:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322649856:12345-0@lo:8:kworker/u8:0.0 Request processed in 106us (283us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316163.293003:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2577 00000100:00080000:2.0:1761316163.293033:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316163.293036:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316163.293039:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba4634d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322649792:0@lo:8:kworker/u8:0.0 00000100:00080000:2.0:1761316163.293046:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316163.293049:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316163.293051:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322649856:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316163.293073:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322649728:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.293080:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322649536:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.293091:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322649920:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316163.293102:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a014551 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322649408:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316163.293204:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033e2b8a8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322649984:12345-0@lo:400:kworker/u8:0.0 Request processed in 1100us (1202us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316163.293209:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea95e939 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322650048:12345-0@lo:400:kworker/u8:0.0 Request processed in 1013us (1290us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316163.293216:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5231 00000100:00100000:0.0:1761316163.293221:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1673 00000100:00100000:3.0:1761316163.293659:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1e2a516 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322650048:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316163.293670:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e1303475 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322649984:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316163.293680:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322649664:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761316164.315514:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316165.320550:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316165.320593:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316165.320626:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316165.320670:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316165.320703:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585088 00000100:00100000:2.0:1761316165.320705:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585216 00000100:00100000:2.0:1761316165.320745:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2578 00000100:00100000:2.0:1761316165.320751:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307585216:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:0.0:1761316165.320762:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585152 00000100:00100000:0.0:1761316165.320785:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5233 00000100:00100000:0.0:1761316165.320798:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a014551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307585088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00010000:02020000:2.0:1761316165.320844:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316165.320868:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a014551 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307585088:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 68us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316165.320877:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5233 00000100:00100000:2.0:1761316165.320885:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307585216:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 132us (215us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316165.320893:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2578 00000100:00100000:0.0:1761316165.320898:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5234 00000100:00100000:0.0:1761316165.320903:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000590ef1c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307585152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316165.320945:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000590ef1c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307585152:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (319us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316165.320952:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5234 00000100:00100000:3.0:1761316165.321055:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585024 00000100:00100000:3.0:1761316165.321088:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1913 00000100:00100000:3.0:1761316165.321093:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307585024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316165.321118:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004526daba pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307585024:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (576us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316165.321124:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1913 00010000:00010000:0.0:1761316165.339517:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316165.851659:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316165.851680:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322650112:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316165.851743:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316165.852082:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650112 00000100:00100000:3.0:1761316165.852135:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3259 00000100:00100000:3.0:1761316165.852142:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322650112:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316165.852172:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316165.852222:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322650112:12345-0@lo:41:osp-pre-1-0.0 Request processed in 80us (479us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316165.852232:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3259 00000100:00100000:1.0:1761316165.852727:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322650112:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316166.365080:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316167.388023:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316168.411530:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316168.411552:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322650240:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316168.411919:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316168.411987:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650240 00000100:00100000:2.0:1761316168.412043:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3260 00000100:00100000:2.0:1761316168.412050:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322650240:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316168.412083:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316168.412138:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322650240:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (249us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316168.412145:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3260 00000100:00100000:3.0:1761316168.412251:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000017a833b2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322650240:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316168.412298:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316168.412413:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316168.412422:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322650176:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316168.412455:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316168.412574:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650176 00000100:00080000:0.0:1761316168.412780:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316168.412785:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316168.412789:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316168.412793:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4410 00000100:00080000:0.0:1761316168.412803:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316168.412806:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316168.412825:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b6d1e3fc x1846869322650560/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316168.412840:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316168.412843:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316168.412844:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316168.412847:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4410 00000100:00080000:0.0:1761316168.412849:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316168.412852:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316168.412868:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ff653b22 x1846869322650624/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316168.412916:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322650496:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316168.412946:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316168.412965:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761316168.412970:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322650304:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316168.413203:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1107 00000100:00100000:3.0:1761316168.413210:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000168f6c92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322650176:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316168.413234:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316168.413268:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000168f6c92 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322650176:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (813us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316168.413277:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1107 00000100:00100000:1.0:1761316168.413293:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322650368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413334:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316168.413377:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003ca09259 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322650752:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.413380:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322650560:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413406:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316168.413428:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316168.413444:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322650176:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316168.413452:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ff653b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322650624:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413464:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761316168.413472:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322650432:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.413478:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316168.413510:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316168.413511:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650560 00000100:00100000:1.0:1761316168.413530:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a5dd2733 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322650816:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.413536:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650624 00000100:00100000:1.0:1761316168.413558:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316168.413561:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2579 00000100:00100000:2.0:1761316168.413567:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322650560:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413596:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650368 00000100:00100000:1.0:1761316168.413622:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650752 00000100:00100000:1.0:1761316168.413638:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650432 00000100:00100000:1.0:1761316168.413652:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5235 00000100:00100000:1.0:1761316168.413656:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006cf388c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322650368:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:2.0:1761316168.413661:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316168.413697:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006cf388c6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322650368:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316168.413706:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5235 00000100:00100000:2.0:1761316168.413710:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322650560:12345-0@lo:8:kworker/u8:2.0 Request processed in 139us (280us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316168.413719:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2579 00000100:00100000:1.0:1761316168.413724:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5236 00000100:00100000:1.0:1761316168.413729:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e1303475 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322650752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.413735:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650816 00000100:00100000:1.0:1761316168.413763:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e1303475 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322650752:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (357us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316168.413766:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1674 00000100:00100000:2.0:1761316168.413773:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1a5999a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322650816:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413775:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5236 00000100:00100000:1.0:1761316168.413791:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5237 00000100:00100000:1.0:1761316168.413795:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1e2a516 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322650432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.413825:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1e2a516 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322650432:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (314us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316168.413831:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5237 00000100:00100000:2.0:1761316168.413841:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1a5999a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322650816:12345-0@lo:400:kworker/u8:2.0 Request processed in 66us (284us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316168.413850:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1674 00000100:00100000:1.0:1761316168.413865:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2580 00000100:00100000:1.0:1761316168.413869:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322650624:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316168.413953:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316168.413977:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322650624:12345-0@lo:8:kworker/u8:2.0 Request processed in 108us (500us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316168.413983:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2580 00000100:00100000:1.0:1761316168.414030:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057310196 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322650432:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414038:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a5dd2733 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322650816:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414046:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322650368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414053:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003ca09259 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322650752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414099:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650496 00000100:00100000:1.0:1761316168.414125:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3261 00000100:00100000:1.0:1761316168.414129:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322650496:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414159:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322650496:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (1214us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316168.414164:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3261 00000100:00080000:1.0:1761316168.414214:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316168.414218:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316168.414221:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b6d1e3fc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322650560:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316168.414230:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316168.414233:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316168.414237:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ff653b22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322650624:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316168.414495:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316168.414525:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002aaef6f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322650688:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316168.414552:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316168.414579:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650304 00000100:00100000:0.0:1761316168.414584:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322650496:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414664:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650688 00000100:00100000:1.0:1761316168.414712:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5238 00000100:00100000:1.0:1761316168.414715:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322650688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316168.414750:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b576fbf3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322650688:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (197us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316168.414755:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5238 00000100:00100000:2.0:1761316168.414868:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1914 00000100:00100000:2.0:1761316168.414874:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322650304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316168.414905:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002aaef6f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322650688:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.414923:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005f9b50ea pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322650304:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (428us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316168.414929:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322650304:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316168.414934:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1914 00010000:00010000:0.0:1761316169.435506:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316170.442426:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316170.442472:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316170.442499:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316170.442512:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585280 00000100:00100000:2.0:1761316170.442574:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1915 00000100:00100000:2.0:1761316170.442581:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307585280:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316170.442594:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585408 00000100:00100000:1.0:1761316170.442629:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585344 00000100:00100000:1.0:1761316170.442651:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5239 00000100:00100000:2.0:1761316170.442654:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000364e1027 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307585280:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 72us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316170.442659:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307585408:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316170.442669:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1915 00000100:00100000:1.0:1761316170.442703:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000053fa0bb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307585408:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 44us (231us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316170.442711:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5239 00000100:00100000:1.0:1761316170.442725:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5240 00000100:00100000:1.0:1761316170.442730:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307585344:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316170.442757:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002118a78f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307585344:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 26us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316170.442763:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5240 00000100:00100000:3.0:1761316170.444938:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316170.445002:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585472 00000100:00100000:2.0:1761316170.445048:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2581 00000100:00100000:2.0:1761316170.445055:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307585472:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761316170.445154:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316170.445211:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307585472:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 155us (277us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316170.445219:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2581 00010000:00010000:3.0:1761316170.459723:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316170.971585:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322650880:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316170.971666:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316170.972782:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650880 00000100:00100000:3.0:1761316170.972839:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3262 00000100:00100000:3.0:1761316170.972845:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322650880:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316170.972891:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316170.972949:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322650880:12345-0@lo:41:osp-pre-1-0.0 Request processed in 102us (1284us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316170.972956:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3262 00000100:00100000:0.0:1761316170.973103:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067246238 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322650880:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316171.483532:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316172.507528:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316173.531915:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322650944:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316173.531998:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316173.532056:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322650944 00000100:00100000:3.0:1761316173.532097:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1108 00000100:00100000:3.0:1761316173.532104:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322650944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:3.0:1761316173.532133:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316173.532151:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322651072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.532178:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067246238 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322650944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 73us (180us total) trans 0 rc 0/0 00000100:00080000:2.0:1761316173.532186:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761316173.532188:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1108 00000100:00080000:2.0:1761316173.532193:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1761316173.532196:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761316173.532198:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316173.532205:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4415 00000100:00080000:2.0:1761316173.532209:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761316173.532214:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316173.532220:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322651200:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.532226:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c43bfbdc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322651328:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316173.532237:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000000386c86 x1846869322651392/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316173.532254:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316173.532258:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316173.532260:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761316173.532265:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:2.0:1761316173.532266:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4415 00000100:00100000:1.0:1761316173.532270:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651072 00000100:00080000:2.0:1761316173.532270:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761316173.532274:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316173.532297:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f2e7988a x1846869322651456/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316173.532303:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1916 00000100:00100000:3.0:1761316173.532313:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002489a343 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322650944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316173.532314:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322651072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.532340:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:3.0:1761316173.532374:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322651136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.532379:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e591f578 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322651072:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (184us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316173.532387:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1916 00000100:00100000:3.0:1761316173.532411:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316173.532418:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322651392:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316173.532430:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322651520:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.532449:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316173.532463:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316173.532469:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322651456:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316173.532489:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322651584:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.532496:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316173.532522:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316173.532557:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651200 00000100:00100000:2.0:1761316173.532577:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651136 00000100:00100000:3.0:1761316173.532583:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651520 00000100:00100000:1.0:1761316173.532589:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3263 00000100:00100000:1.0:1761316173.532595:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322651200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316173.532619:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651584 00000100:00100000:1.0:1761316173.532635:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322651200:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (373us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316173.532641:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3263 00000100:00100000:0.0:1761316173.533227:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322651008:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316173.533229:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5241 00000100:00100000:3.0:1761316173.533237:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322651136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533270:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316173.533280:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322651136:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (869us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316173.533287:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5241 00000100:00100000:0.0:1761316173.533293:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322651264:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.533303:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5242 00000100:00100000:3.0:1761316173.533307:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322651520:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533320:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316173.533341:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000aca01809 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322651648:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.533369:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002489a343 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322651520:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (907us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316173.533377:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5242 00000100:00100000:0.0:1761316173.533389:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316173.533392:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651264 00000100:00100000:3.0:1761316173.533412:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5243 00000100:00100000:3.0:1761316173.533417:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322651584:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533432:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5244 00000100:00100000:0.0:1761316173.533438:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322651264:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.533447:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e083254 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322651584:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (925us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316173.533453:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5243 00000100:00100000:3.0:1761316173.533475:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651392 00000100:00100000:0.0:1761316173.533483:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6e8631e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322651264:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (163us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316173.533491:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5244 00000100:00100000:3.0:1761316173.533498:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651456 00000100:00100000:0.0:1761316173.533519:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651648 00000100:00100000:3.0:1761316173.533522:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2582 00000100:00100000:3.0:1761316173.533527:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322651392:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533550:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1675 00000100:00100000:0.0:1761316173.533555:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322651648:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533583:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322651648:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (196us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316173.533588:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1675 00000100:00100000:0.0:1761316173.533599:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2583 00000100:00100000:0.0:1761316173.533602:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322651456:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:3.0:1761316173.533609:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316173.533657:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322651392:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (1209us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316173.533664:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2582 00010000:02020000:0.0:1761316173.533682:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00080000:3.0:1761316173.533695:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316173.533699:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316173.533704:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322651392:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761316173.533715:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316173.533717:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316173.533720:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322651456:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316173.533734:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322651456:12345-0@lo:8:kworker/u8:0.0 Request processed in 114us (1223us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316173.533740:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2583 00000100:00100000:3.0:1761316173.534012:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651008 00000100:00100000:3.0:1761316173.534035:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3264 00000100:00100000:3.0:1761316173.534039:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322651008:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316173.534059:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316173.536091:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000045ef189a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322651264:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316173.536123:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000aca01809 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322651648:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316173.536141:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316173.536169:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322651008:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316173.536170:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322651008:12345-0@lo:41:osp-pre-0-1.0 Request processed in 2126us (2897us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316173.536183:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006af8133c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322651072:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.536186:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3264 00000100:00100000:3.0:1761316173.536219:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651328 00000100:00100000:3.0:1761316173.536256:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3265 00000100:00100000:3.0:1761316173.536261:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322651328:12345-0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316173.536270:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316173.536275:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322651584:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.536292:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322651136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316173.536307:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eaa26ccb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322651520:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.536318:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322651328:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (178us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316173.536323:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322651200:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316173.536328:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3265 00000100:00100000:3.0:1761316173.536409:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c43bfbdc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322651328:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316174.555516:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316175.562110:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316175.562159:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316175.562203:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316175.562240:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316175.562520:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585536 00000100:00100000:1.0:1761316175.562581:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1917 00000100:00100000:1.0:1761316175.562589:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307585536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316175.562654:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008d587bc6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307585536:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 66us (495us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316175.562662:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1917 00000100:00100000:1.0:1761316175.562699:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585664 00000100:00100000:1.0:1761316175.562724:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585600 00000100:00100000:1.0:1761316175.562741:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5245 00000100:00100000:1.0:1761316175.562746:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307585664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316175.562777:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aef6c9f8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307585664:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (583us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316175.562785:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5245 00000100:00100000:1.0:1761316175.562797:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5246 00000100:00100000:1.0:1761316175.562801:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307585600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316175.562825:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aca01809 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307585600:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (585us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316175.562831:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5246 00000100:00100000:0.0:1761316175.564177:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585728 00000100:00100000:0.0:1761316175.564229:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2584 00000100:00100000:0.0:1761316175.564235:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006380a1e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307585728:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761316175.564332:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316175.564430:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006380a1e6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:2262:x1846869307585728:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 193us (2327us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316175.564439:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2584 00010000:00010000:0.0:1761316175.579575:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316176.094653:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316176.094672:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322651712:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316176.094739:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316176.094846:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651712 00000100:00100000:3.0:1761316176.094897:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3266 00000100:00100000:3.0:1761316176.094903:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322651712:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316176.094933:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316176.094986:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322651712:12345-0@lo:41:osp-pre-1-0.0 Request processed in 79us (245us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316176.094994:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3266 00000100:00100000:1.0:1761316176.095110:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322651712:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316176.603486:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316177.627538:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316178.651495:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316178.651600:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322651840:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316178.651670:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.651711:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316178.651719:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322651776:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316178.651744:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.651775:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651776 00000100:00100000:0.0:1761316178.651813:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3267 00000100:00100000:0.0:1761316178.651847:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322651776:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316178.651876:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316178.651914:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322651776:12345-0@lo:41:osp-pre-0-1.0 Request processed in 67us (169us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316178.651920:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3267 00000100:00100000:0.0:1761316178.651959:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007ae5eee8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322651776:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316178.652034:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651840 00000100:00100000:3.0:1761316178.652069:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1918 00000100:00100000:3.0:1761316178.652073:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322651840:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316178.652109:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000901feca2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322651840:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (442us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316178.652116:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1918 00000100:00100000:3.0:1761316178.652148:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322651840:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316178.652286:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322651904:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316178.652313:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316178.652537:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322651968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.652565:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.652568:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651904 00000100:00100000:0.0:1761316178.652599:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322651968 00000100:00100000:2.0:1761316178.652607:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5247 00000100:00100000:2.0:1761316178.652611:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322651904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316178.652617:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5248 00000100:00100000:0.0:1761316178.652620:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322651968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316178.652652:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006af8133c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322651968:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (86us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316178.652658:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5248 00000100:00100000:0.0:1761316178.652676:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322652032:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316178.652700:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.652742:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652032 00000100:00100000:0.0:1761316178.652765:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1109 00000100:00100000:0.0:1761316178.652769:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322652032:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316178.652787:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316178.652814:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003ca09259 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322652032:12345-0@lo:13:osp-pre-1-1.0 Request processed in 45us (114us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316178.652830:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1109 00000100:00100000:0.0:1761316178.652857:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034b089b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322652032:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316178.652982:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ae5eee8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322651904:12345-0@lo:400:kworker/u8:2.0 Request processed in 370us (668us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316178.652989:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5247 00000100:00080000:1.0:1761316178.652996:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316178.653001:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316178.653005:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316178.653008:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4421 00000100:00080000:1.0:1761316178.653011:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761316178.653013:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322651968:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316178.653014:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316178.653032:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b893325d x1846869322652160/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316178.653045:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316178.653048:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316178.653050:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316178.653053:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4421 00000100:00080000:1.0:1761316178.653054:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316178.653057:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316178.653067:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007f104f12 x1846869322652224/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316178.653104:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652416:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653131:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316178.653148:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316178.653153:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652096:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653175:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316178.653175:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322652160:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653197:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652416 00000100:00100000:2.0:1761316178.653198:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316178.653211:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322652224:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653218:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652160 00000100:00100000:2.0:1761316178.653230:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316178.653232:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1676 00000100:00100000:1.0:1761316178.653236:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653252:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652224 00000100:00100000:1.0:1761316178.653265:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6d1e3fc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652416:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (134us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316178.653270:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2585 00000100:00100000:1.0:1761316178.653273:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1676 00000100:00100000:2.0:1761316178.653273:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322652160:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653282:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2586 00000100:00100000:1.0:1761316178.653284:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652224:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316178.653461:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:2.0:1761316178.653462:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316178.653495:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322652160:12345-0@lo:8:kworker/u8:2.0 Request processed in 221us (298us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316178.653502:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2585 00000100:00100000:0.0:1761316178.653528:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652096 00000100:00080000:2.0:1761316178.653529:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316178.653531:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316178.653534:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322652160:0@lo:8:kworker/u8:2.0 00000100:00080000:2.0:1761316178.653541:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316178.653543:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316178.653545:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322652224:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316178.653549:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3268 00000100:00100000:0.0:1761316178.653553:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652096:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653560:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322652352:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653636:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.653639:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652096:12345-0@lo:400:kworker/u8:2.0 Request processed in 86us (464us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316178.653646:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3268 00000100:00100000:2.0:1761316178.653651:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316178.653655:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003e303c6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322652288:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653676:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316178.653677:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f1a5999a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652096:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316178.653685:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652416:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653703:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652352 00000100:00100000:2.0:1761316178.653723:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652288 00000100:00100000:3.0:1761316178.653746:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000631748a1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322651904:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653794:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5249 00000100:00100000:3.0:1761316178.653797:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5250 00000100:00100000:2.0:1761316178.653798:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322652352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316178.653800:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322652288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316178.653840:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000825eb04a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322652352:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (204us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316178.653840:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f32569ca pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322652288:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (163us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316178.653846:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5250 00000100:00100000:2.0:1761316178.653849:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5249 00000100:00100000:3.0:1761316178.653882:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003e303c6b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322652288:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316178.653891:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322652352:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316178.653934:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652224:12345-0@lo:8:kworker/u8:2.0 Request processed in 647us (703us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316178.653941:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2586 00010000:00010000:0.0:1761316179.675566:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316180.681125:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316180.683336:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316180.683736:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585856 00000100:00100000:0.0:1761316180.683817:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5251 00000100:00100000:0.0:1761316180.683824:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307585856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316180.683833:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316180.683900:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c43bfbdc pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307585856:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 75us (2791us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316180.683908:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5251 00000100:00100000:0.0:1761316180.683925:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585920 00000100:00100000:0.0:1761316180.683944:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5252 00000100:00100000:0.0:1761316180.683948:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307585920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316180.683972:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000eaa26ccb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307585920:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 23us (140us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316180.683978:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5252 00000100:00100000:3.0:1761316180.684101:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316180.684905:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585792 00000100:00100000:1.0:1761316180.684943:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1919 00000100:00100000:1.0:1761316180.684948:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307585792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316180.685631:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307585984 00000100:00100000:1.0:1761316180.685898:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e643545 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307585792:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 947us (2566us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316180.685907:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1919 00000100:00100000:3.0:1761316180.686540:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2587 00000100:00100000:3.0:1761316180.686545:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307585984:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316180.686637:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316180.686704:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000036dbb695 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307585984:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 157us (2603us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316180.686715:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2587 00010000:00010000:0.0:1761316180.700572:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316181.211668:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316181.211769:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652480:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316181.211846:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316181.211940:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652480 00000100:00100000:0.0:1761316181.211992:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3269 00000100:00100000:0.0:1761316181.211998:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652480:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316181.212031:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316181.212084:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322652480:12345-0@lo:41:osp-pre-1-0.0 Request processed in 84us (238us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316181.212091:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3269 00000100:00100000:1.0:1761316181.212170:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f2e7988a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322652480:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316181.723672:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316182.749232:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316183.771535:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316183.771708:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316183.771730:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322652544:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316183.771824:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316183.771910:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652544 00000100:00100000:2.0:1761316183.771960:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1110 00000100:00100000:2.0:1761316183.771966:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322652544:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316183.771993:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316183.772035:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a5dd2733 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322652544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 69us (212us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316183.772042:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1110 00000100:00100000:0.0:1761316183.772182:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322652608:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316183.772211:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316183.772253:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322652544:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316183.772319:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652608 00000100:00100000:3.0:1761316183.772366:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3270 00000100:00100000:3.0:1761316183.772370:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322652608:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316183.772391:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316183.772435:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322652608:12345-0@lo:41:osp-pre-0-1.0 Request processed in 64us (224us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316183.772441:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3270 00000100:00100000:2.0:1761316183.772507:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000386c86 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322652608:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316183.777975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000851117f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322652672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.778031:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322652800:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778045:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316183.778065:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316183.778071:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316183.778077:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322652736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778095:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316183.778124:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652672 00000100:00100000:1.0:1761316183.778158:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1920 00000100:00100000:1.0:1761316183.778162:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322652672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778198:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4cff5d3 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322652672:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (154us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316183.778204:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1920 00000100:00100000:1.0:1761316183.778229:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652800 00000100:00100000:1.0:1761316183.778246:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652736 00000100:00100000:1.0:1761316183.778258:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5253 00000100:00100000:1.0:1761316183.778261:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003e303c6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322652800:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778673:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003e303c6b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322652800:12345-0@lo:400:kworker/u8:2.0 Request processed in 410us (607us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316183.778680:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5253 00000100:00100000:1.0:1761316183.778693:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5254 00000100:00100000:1.0:1761316183.778697:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322652736:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778722:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057310196 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322652736:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (627us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316183.778727:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5254 00000100:00100000:1.0:1761316183.778756:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322652736:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.778765:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000851117f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322652672:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.778946:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322652800:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.779090:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322652864:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.779119:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316183.780312:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316183.780320:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316183.780323:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316183.780329:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4426 00000100:00080000:0.0:1761316183.780332:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316183.780337:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316183.781974:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652864 00000100:00100000:1.0:1761316183.782038:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3271 00000100:00100000:1.0:1761316183.782043:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322652864:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.782125:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322652864:12345-0@lo:400:kworker/u8:2.0 Request processed in 81us (3005us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316183.782132:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3271 00000100:00100000:2.0:1761316183.782404:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322652864:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316183.782779:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005e264e5f x1846869322652928/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316183.782815:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316183.782820:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316183.782822:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316183.782863:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4426 00000100:00080000:0.0:1761316183.782865:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316183.782869:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316183.782884:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005e6dc415 x1846869322652992/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316183.782952:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316183.782962:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322653184:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316183.783001:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316183.783030:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653184 00000100:00100000:0.0:1761316183.783060:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1677 00000100:00100000:0.0:1761316183.783064:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322653184:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316183.783098:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322653184:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (97us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316183.783104:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1677 00000100:00100000:0.0:1761316183.783131:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005fdf8940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322653184:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.783880:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322653056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.783921:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316183.783956:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316183.783962:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322653120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.783986:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316183.784029:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653056 00000100:00100000:1.0:1761316183.784059:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653120 00000100:00100000:1.0:1761316183.784078:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5255 00000100:00100000:1.0:1761316183.784083:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322653056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.784412:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322652928:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316183.784441:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316183.784455:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322652992:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316183.784471:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316183.784509:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5256 00000100:00100000:2.0:1761316183.784513:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322653120:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316183.784549:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a0897acb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322653120:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (562us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316183.784555:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5256 00000100:00100000:2.0:1761316183.784589:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d93c9bea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322653120:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316183.786013:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322653056:12345-0@lo:400:kworker/u8:2.0 Request processed in 1925us (2087us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316183.786025:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5255 00000100:00100000:0.0:1761316183.786138:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652928 00000100:00100000:0.0:1761316183.786175:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322652992 00000100:00100000:0.0:1761316183.786200:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2588 00000100:00100000:0.0:1761316183.786204:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006b9872de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652928:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761316183.786292:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316183.786334:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006b9872de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652928:12345-0@lo:8:kworker/u8:2.0 Request processed in 128us (1893us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316183.786340:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2588 00000100:00100000:0.0:1761316183.786371:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2589 00000100:00100000:0.0:1761316183.786374:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652992:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761316183.786456:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316183.786478:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000631748a1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322652992:12345-0@lo:8:kworker/u8:2.0 Request processed in 103us (2007us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316183.786484:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2589 00000100:00080000:0.0:1761316183.786521:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316183.786525:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316183.786529:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e264e5f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322652928:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761316183.786538:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316183.786540:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316183.786543:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005e6dc415 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322652992:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316183.787176:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322653056:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316184.795623:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316185.800993:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316185.801048:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316185.801076:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586112 00000100:00100000:3.0:1761316185.801086:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316185.801149:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586176 00000100:00100000:2.0:1761316185.801194:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5257 00000100:00100000:2.0:1761316185.801205:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307586176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316185.801274:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2e7988a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307586176:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 72us (189us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316185.801281:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5257 00000100:00100000:1.0:1761316185.801674:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586048 00000100:00100000:2.0:1761316185.802273:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1921 00000100:00100000:2.0:1761316185.802280:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307586048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316185.802321:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008764c92c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307586048:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (1274us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316185.802328:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1921 00000100:00100000:0.0:1761316185.802669:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5258 00000100:00100000:0.0:1761316185.802675:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307586112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316185.802710:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000386c86 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307586112:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 35us (1725us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316185.802716:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5258 00000100:00100000:2.0:1761316185.803573:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316185.804210:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586240 00000100:00100000:1.0:1761316185.804252:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2590 00000100:00100000:1.0:1761316185.804258:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1a5999a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307586240:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316185.804398:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316185.804458:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1a5999a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307586240:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 198us (887us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316185.804465:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2590 00010000:00010000:0.0:1761316185.820229:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316186.331561:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316186.331664:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322653248:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316186.331754:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316186.335037:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653248 00000100:00100000:3.0:1761316186.335164:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3272 00000100:00100000:3.0:1761316186.335171:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000628f9f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322653248:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316186.335211:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316186.335401:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000628f9f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322653248:12345-0@lo:41:osp-pre-1-0.0 Request processed in 229us (3646us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316186.335408:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3272 00000100:00100000:2.0:1761316186.335811:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027579ed2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322653248:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316186.843622:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316187.870076:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316188.891979:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322653312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316188.892044:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316188.892094:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653312 00000100:00100000:2.0:1761316188.892129:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1111 00000100:00100000:2.0:1761316188.892134:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003d6c654 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322653312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316188.892159:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316188.892189:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003d6c654 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322653312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 54us (146us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316188.892195:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1111 00010000:00010000:3.0:1761316188.892224:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316188.892239:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322653376:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316188.892242:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b893325d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322653312:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316188.892282:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322653440:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.892306:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316188.892308:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316188.892313:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316188.892317:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316188.892322:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316188.892325:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4431 00000100:00080000:1.0:1761316188.892329:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761316188.892330:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322653632:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316188.892333:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316188.892368:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322653696:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316188.892378:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000086cdb3 x1846869322653760/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316188.892383:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316188.892393:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316188.892398:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316188.892400:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316188.892402:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761316188.892403:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00080000:1.0:1761316188.892406:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4431 00000100:00080000:1.0:1761316188.892408:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761316188.892409:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322653568:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316188.892411:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316188.892422:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316188.892423:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000ad86ef88 x1846869322653824/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316188.892431:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322653504:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.892438:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316188.892457:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316188.892463:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316188.892467:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322654016:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316188.892474:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653440 00000100:00100000:2.0:1761316188.892483:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316188.892491:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322653888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.892498:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316188.892502:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1922 00000100:00100000:1.0:1761316188.892507:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322653440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.892520:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322653952:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316188.892524:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316188.892550:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316188.892562:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653376 00000100:00100000:2.0:1761316188.892591:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653696 00000100:00100000:3.0:1761316188.893156:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653568 00000100:00100000:3.0:1761316188.893177:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3273 00000100:00100000:3.0:1761316188.893180:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322653376:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316188.893202:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316188.893238:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000027579ed2 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322653376:12345-0@lo:41:osp-pre-0-1.0 Request processed in 56us (933us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893244:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3273 00000100:00100000:3.0:1761316188.893256:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3274 00000100:00100000:3.0:1761316188.893260:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322653696:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893289:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b893325d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322653696:12345-0@lo:400:kworker/u8:0.0 Request processed in 29us (895us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893295:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3274 00000100:00100000:3.0:1761316188.893305:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3275 00000100:00100000:3.0:1761316188.893308:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322653568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316188.893314:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005bd266a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322653440:12345-0@lo:400:kworker/u8:0.0 Request processed in 807us (996us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316188.893320:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1922 00000100:00100000:3.0:1761316188.893328:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004e9b55f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322653568:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (892us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893332:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3275 00000100:00100000:3.0:1761316188.893367:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653632 00000100:00100000:1.0:1761316188.893387:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322653760:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893388:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653504 00000100:00100000:3.0:1761316188.893401:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653888 00000100:00100000:3.0:1761316188.893410:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653952 00000100:00100000:1.0:1761316188.893412:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316188.893420:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5259 00000100:00100000:3.0:1761316188.893423:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322653632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316188.893426:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322653824:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316188.893442:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316188.893472:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654016 00000100:00100000:1.0:1761316188.893491:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653760 00000100:00100000:1.0:1761316188.893504:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322653824 00000100:00100000:1.0:1761316188.893517:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1678 00000100:00100000:1.0:1761316188.893520:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322654016:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893527:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009e135009 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322653632:12345-0@lo:400:kworker/u8:0.0 Request processed in 103us (1145us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893533:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5259 00000100:00100000:3.0:1761316188.893544:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5260 00000100:00100000:3.0:1761316188.893547:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f887dab3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322653504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893569:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f887dab3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322653504:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (1109us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893575:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5260 00000100:00100000:3.0:1761316188.893583:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5261 00000100:00100000:3.0:1761316188.893586:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7a799d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322653888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893606:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7a799d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322653888:12345-0@lo:400:kworker/u8:0.0 Request processed in 19us (1086us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893611:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5261 00000100:00100000:3.0:1761316188.893619:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5262 00000100:00100000:3.0:1761316188.893622:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322653952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893643:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000096cf7252 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322653952:12345-0@lo:400:kworker/u8:0.0 Request processed in 20us (1094us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316188.893647:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5262 00000100:00100000:3.0:1761316188.893681:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322653952:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893690:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322653568:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893699:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d8e4afe2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322653632:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893710:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024e9f3ae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322653376:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316188.893790:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bf1342a6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322653888:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893797:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2591 00000100:00100000:3.0:1761316188.893802:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322653760:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316188.893805:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d1eea702 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322653504:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316188.893815:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004fe06698 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322653696:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316188.893823:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322653440:0@lo:400:kworker/u8:0.0 00010000:02020000:3.0:1761316188.893887:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316188.893926:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000007e02a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322653760:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (514us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316188.893930:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2592 00000100:00100000:2.0:1761316188.893941:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322653824:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316188.893941:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2591 00000100:00080000:3.0:1761316188.893996:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316188.894000:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316188.894003:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000086cdb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322653760:0@lo:8:kworker/u8:0.0 00010000:02020000:2.0:1761316188.894028:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316188.894071:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c77a9413 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322653824:12345-0@lo:8:kworker/u8:0.0 Request processed in 133us (628us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316188.894078:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2592 00000100:00080000:3.0:1761316188.894147:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316188.894151:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316188.894154:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322653824:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316188.897289:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033ca1fdd pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322654016:12345-0@lo:400:kworker/u8:0.0 Request processed in 3766us (4790us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316188.897301:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1678 00000100:00100000:3.0:1761316188.897407:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322654016:0@lo:400:kworker/u8:0.0 00010000:00010000:2.0:1761316189.915502:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316190.920101:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316190.920153:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316190.920396:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586368 00000100:00100000:0.0:1761316190.920450:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5263 00000100:00100000:0.0:1761316190.920457:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307586368:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316190.920527:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d8880f6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307586368:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 70us (433us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316190.920537:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5263 00000100:00100000:3.0:1761316190.921377:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316190.921474:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586304 00000100:00100000:1.0:1761316190.921505:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1923 00000100:00100000:1.0:1761316190.921510:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307586304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316190.921552:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000079989f7c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307586304:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 41us (1399us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316190.921559:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1923 00000100:00100000:3.0:1761316190.921753:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316190.921822:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586432 00000100:00100000:0.0:1761316190.921960:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5264 00000100:00100000:0.0:1761316190.921965:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307586432:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316190.922003:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000740059ce pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307586432:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 37us (628us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316190.922009:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5264 00000100:00100000:1.0:1761316190.922314:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586496 00000100:00100000:1.0:1761316190.922363:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2593 00000100:00100000:1.0:1761316190.922368:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307586496:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761316190.922465:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316190.922507:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000086cdb3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307586496:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 138us (754us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316190.922515:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2593 00010000:00010000:0.0:1761316190.939560:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316191.451930:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322654080:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316191.452024:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316191.452390:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654080 00000100:00100000:3.0:1761316191.452463:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3276 00000100:00100000:3.0:1761316191.452471:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0ca0bde pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322654080:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316191.452513:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316191.452749:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0ca0bde pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322654080:12345-0@lo:41:osp-pre-1-0.0 Request processed in 277us (726us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316191.452758:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3276 00000100:00100000:2.0:1761316191.452845:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004353fd95 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322654080:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316191.963651:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316192.989081:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316194.011598:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316194.011607:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1761316194.011609:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654336:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316194.011612:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316194.011620:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4436 00000100:00080000:1.0:1761316194.011625:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316194.011633:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316194.011658:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005af8062f x1846869322654528/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316194.011673:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316194.011678:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316194.011679:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316194.011683:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4436 00000100:00080000:1.0:1761316194.011685:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316194.011689:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316194.011697:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316194.011701:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f6f0cd4a x1846869322654592/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316194.011754:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c90fb5e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322654720:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.011787:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316194.011825:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1761316194.011831:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000039391167 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654144:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316194.011859:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316194.011876:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b10b1dae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322654400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.011881:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322654208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.011899:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316194.011917:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.011944:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316194.012011:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654720 00000100:00100000:0.0:1761316194.012038:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654336 00000100:00100000:1.0:1761316194.012062:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654400 00000100:00100000:0.0:1761316194.012084:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3277 00000100:00100000:1.0:1761316194.012088:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5265 00000100:00100000:0.0:1761316194.012093:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c74a1dc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322654336:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316194.012096:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e341f863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322654720:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.012928:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d940ebd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322654464:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.012939:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316194.012962:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316194.012984:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322654848:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013000:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654144 00000100:00100000:3.0:1761316194.013005:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316194.013024:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761316194.013031:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322654272:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013050:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1112 00000100:00100000:3.0:1761316194.013054:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316194.013057:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1f6a7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322654144:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316194.013070:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3f414de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322654656:0@lo:400:kworker/u8:0.0 00002000:00000024:2.0:1761316194.013087:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316194.013092:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316194.013134:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1f6a7de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322654144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 78us (1276us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316194.013143:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1112 00000100:00100000:3.0:1761316194.013152:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654208 00000100:00100000:2.0:1761316194.013176:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654272 00000100:00100000:3.0:1761316194.013191:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1924 00000100:00100000:3.0:1761316194.013196:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322654208:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013200:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654656 00000100:00100000:2.0:1761316194.013218:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5266 00000100:00100000:2.0:1761316194.013223:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322654400:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.013229:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007a3c89e1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322654208:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (1294us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.013237:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1924 00000100:00100000:2.0:1761316194.013262:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f4365f07 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322654400:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (1363us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.013271:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5267 00000100:00100000:3.0:1761316194.013275:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322654272:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.013308:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024e9f3ae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322654272:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316194.013364:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e341f863 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322654720:12345-0@lo:400:kworker/u8:0.0 Request processed in 1270us (1580us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316194.013464:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5266 00000100:00100000:3.0:1761316194.013475:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5267 00000100:00100000:2.0:1761316194.013488:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5268 00000100:00100000:2.0:1761316194.013493:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322654656:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013536:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d8e4afe2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322654656:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (445us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.013540:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000357a3bb2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322654272:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013547:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5268 00000100:00100000:3.0:1761316194.013558:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3f414de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322654656:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.013569:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322654208:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013574:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654784 00000100:00100000:3.0:1761316194.013591:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:3.0:1761316194.013600:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322654912:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316194.013601:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654848 00000100:00100000:2.0:1761316194.013623:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1679 00000100:00100000:2.0:1761316194.013629:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322654784:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.013631:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316194.013661:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1680 00000100:00100000:2.0:1761316194.013670:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba4634d0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322654784:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (730us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.013670:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322654848:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.013683:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1679 00000100:00100000:2.0:1761316194.013719:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654464 00000100:00100000:3.0:1761316194.013719:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1eea702 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322654848:12345-0@lo:400:kworker/u8:0.0 Request processed in 42us (704us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.013727:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1680 00000100:00100000:2.0:1761316194.013739:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654912 00000100:00100000:2.0:1761316194.013756:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3278 00000100:00100000:2.0:1761316194.013761:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322654464:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316194.013775:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5265 00000100:00100000:2.0:1761316194.013793:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004fe06698 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322654464:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (833us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316194.013825:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322654528:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316194.013851:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316194.013863:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322654592:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316194.013879:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316194.013900:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654528 00000100:00100000:1.0:1761316194.013915:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654592 00000100:00100000:1.0:1761316194.013926:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2594 00000100:00100000:1.0:1761316194.013930:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322654528:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316194.014025:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316194.014053:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d729081 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322654528:12345-0@lo:8:kworker/u8:0.0 Request processed in 123us (202us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316194.014060:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2594 00000100:00100000:1.0:1761316194.014071:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2595 00000100:00100000:1.0:1761316194.014074:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000031d3c62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322654592:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316194.014104:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c74a1dc7 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322654336:12345-0@lo:400:kworker/u8:0.0 Request processed in 2010us (2408us total) trans 0 rc 0/0 00010000:02020000:1.0:1761316194.014158:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316194.014177:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000031d3c62d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322654592:12345-0@lo:8:kworker/u8:0.0 Request processed in 102us (298us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316194.014183:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2595 00000100:00080000:1.0:1761316194.014206:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316194.014210:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316194.014213:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322654528:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761316194.014220:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316194.014228:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316194.014231:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322654592:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014421:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3279 00000100:00100000:3.0:1761316194.014427:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322654912:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316194.014451:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316194.014479:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004353fd95 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322654912:12345-0@lo:41:osp-pre-0-1.0 Request processed in 53us (852us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316194.014485:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3279 00000100:00100000:3.0:1761316194.014511:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ef784d0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322654912:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316194.014521:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d940ebd0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322654464:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014529:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322654848:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014584:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000039391167 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654144:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316194.014594:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b10b1dae pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322654400:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014602:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654784:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014612:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c90fb5e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322654720:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316194.014619:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000327d56cf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322654336:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316194.015238:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3278 00000100:00100000:0.0:1761316194.015304:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3277 00010000:00010000:0.0:1761316194.015414:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316195.036481:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316196.040122:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316196.040194:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316196.040228:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586624 00000100:00100000:3.0:1761316196.040232:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316196.040270:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316196.040340:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586560 00000100:00100000:1.0:1761316196.040667:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586688 00000100:00100000:1.0:1761316196.040706:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5269 00000100:00100000:1.0:1761316196.040715:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307586624:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316196.040823:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002ef784d0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307586624:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 107us (712us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316196.040837:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5269 00000100:00100000:1.0:1761316196.040862:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5270 00000100:00100000:1.0:1761316196.040867:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307586688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316196.040894:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000327d56cf pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307586688:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 27us (665us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316196.040901:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5270 00000100:00100000:1.0:1761316196.040949:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586752 00000100:00100000:1.0:1761316196.040971:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2596 00000100:00100000:1.0:1761316196.040974:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307586752:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761316196.041070:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316196.041104:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307586752:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 128us (834us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316196.041112:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2596 00000100:00100000:0.0:1761316196.042753:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1925 00000100:00100000:0.0:1761316196.042769:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307586560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316196.042872:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000080a45f24 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307586560:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 106us (2679us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316196.042881:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1925 00010000:00010000:2.0:1761316196.060023:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316196.571761:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322654976:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316196.571867:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316196.572133:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322654976 00000100:00100000:1.0:1761316196.572206:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3280 00000100:00100000:1.0:1761316196.572214:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000039391167 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322654976:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316196.572261:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316196.572330:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000039391167 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322654976:12345-0@lo:41:osp-pre-1-0.0 Request processed in 115us (463us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316196.572338:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3280 00000100:00100000:2.0:1761316196.572512:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e7587896 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322654976:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316197.083530:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316198.108193:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316199.131787:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316199.132092:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322655040:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316199.132162:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316199.132169:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1761316199.132173:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008a8e3ffb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655168:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316199.132174:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316199.132184:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4441 00000100:00080000:1.0:1761316199.132189:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316199.132195:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316199.132199:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316199.132225:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e2fc70ab x1846869322655360/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316199.132238:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9cdf97f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322655232:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316199.132244:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316199.132247:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316199.132249:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316199.132253:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4441 00000100:00080000:1.0:1761316199.132255:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316199.132258:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316199.132266:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316199.132275:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004acac90c x1846869322655424/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316199.132286:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316199.132323:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322655616:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.132333:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c90fb5e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322655552:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.132382:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e2fc70ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322655360:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316199.132394:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316199.132423:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316199.132428:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1761316199.132437:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069b4f1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322655296:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.132450:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761316199.132460:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322655104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.132487:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316199.132508:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322655488:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.132522:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316199.132546:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316199.132558:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322655424:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316199.132600:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316199.132702:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655616 00000100:00100000:0.0:1761316199.132752:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655360 00000100:00100000:0.0:1761316199.132765:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655424 00000100:00100000:0.0:1761316199.132780:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1681 00000100:00100000:0.0:1761316199.132789:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322655616:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.132804:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2597 00000100:00100000:1.0:1761316199.132813:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d940ebd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322655360:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316199.132844:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322655616:12345-0@lo:400:kworker/u8:0.0 Request processed in 56us (449us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316199.132852:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1681 00000100:00100000:0.0:1761316199.132870:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2598 00000100:00100000:0.0:1761316199.132873:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322655424:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316199.132972:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316199.133029:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d940ebd0 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322655360:12345-0@lo:8:kworker/u8:0.0 Request processed in 216us (510us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316199.133037:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2597 00010000:02020000:0.0:1761316199.133051:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316199.133075:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655232 00000100:00100000:0.0:1761316199.133097:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3f414de pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322655424:12345-0@lo:8:kworker/u8:0.0 Request processed in 221us (502us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316199.133107:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2598 00000100:00100000:1.0:1761316199.133129:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655552 00000100:00100000:0.0:1761316199.133163:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655168 00000100:00100000:1.0:1761316199.133164:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655104 00000100:00100000:1.0:1761316199.133209:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655488 00000100:00100000:0.0:1761316199.133221:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3281 00000100:00100000:0.0:1761316199.133227:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322655168:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.133239:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5271 00000100:00100000:1.0:1761316199.133246:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322655232:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.133269:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059de7950 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322655168:12345-0@lo:400:kworker/u8:0.0 Request processed in 39us (1007us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316199.133281:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3281 00000100:00100000:1.0:1761316199.133310:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ad86ef88 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322655232:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (1026us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316199.133323:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5271 00000100:00100000:0.0:1761316199.133380:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008a8e3ffb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655168:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.133387:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5272 00000100:00100000:1.0:1761316199.133394:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322655552:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.133466:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322655552:12345-0@lo:400:kworker/u8:0.0 Request processed in 72us (1047us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316199.133473:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5272 00000100:00100000:1.0:1761316199.133490:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5273 00000100:00100000:1.0:1761316199.133507:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322655104:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.133521:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5274 00000100:00100000:0.0:1761316199.133534:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322655488:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.133548:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322655104:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (1059us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316199.133556:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5273 00000100:00100000:0.0:1761316199.133598:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322655488:12345-0@lo:400:kworker/u8:0.0 Request processed in 63us (1052us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316199.133608:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5274 00000100:00080000:1.0:1761316199.133614:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316199.133619:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316199.133626:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e2fc70ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322655360:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761316199.133642:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316199.133645:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316199.133650:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322655424:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316199.133691:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005af8062f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322655104:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.133710:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b867a88f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322655488:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316199.133725:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c90fb5e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322655552:0@lo:400:kworker/u8:0.0 00010000:00010000:1.0:1761316199.133727:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316199.134365:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655040 00000100:00100000:3.0:1761316199.134413:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1926 00000100:00100000:3.0:1761316199.134423:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322655040:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316199.134428:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316199.134905:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316199.134919:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655744:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316199.135371:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322655680:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135380:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316199.135427:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316199.135439:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655296 00000100:00100000:2.0:1761316199.135470:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002eb7ad1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322655616:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135479:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655744 00000100:00100000:2.0:1761316199.135492:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9cdf97f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322655232:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135503:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3282 00000100:00100000:1.0:1761316199.135509:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000098d5ea39 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322655296:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316199.135515:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f40e0c10 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322655040:12345-0@lo:400:kworker/u8:0.0 Request processed in 1094us (3316us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316199.135517:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6f0cd4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322655040:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316199.135531:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1926 00000100:00100000:1.0:1761316199.135555:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000098d5ea39 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322655296:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (1135us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316199.135564:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3282 00000100:00100000:2.0:1761316199.135567:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069b4f1dd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322655296:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135585:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3283 00000100:00100000:3.0:1761316199.135588:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655680 00000100:00100000:1.0:1761316199.135591:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322655744:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316199.135623:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316199.135627:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1682 00000100:00100000:2.0:1761316199.135633:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322655680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135687:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e2fc70ab pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322655744:12345-0@lo:41:osp-pre-0-1.0 Request processed in 78us (291us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316199.135693:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322655680:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (267us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316199.135696:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3283 00000100:00100000:1.0:1761316199.135728:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004acac90c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655744:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316199.135880:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655808:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316199.135902:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322655680:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316199.135919:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316199.136052:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1682 00000100:00100000:0.0:1761316199.136292:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655808 00000100:00100000:0.0:1761316199.138413:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1113 00000100:00100000:0.0:1761316199.138425:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004acac90c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322655808:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316199.138463:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316199.138548:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004acac90c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322655808:12345-0@lo:13:osp-pre-1-1.0 Request processed in 125us (2634us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316199.138555:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1113 00000100:00100000:1.0:1761316199.142763:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322655808:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316200.155634:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316201.160101:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316201.160194:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316201.160200:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586880 00000100:00100000:3.0:1761316201.160229:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316201.160279:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586944 00000100:00100000:3.0:1761316201.160298:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5275 00000100:00100000:3.0:1761316201.160306:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307586880:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316201.160316:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5276 00000100:00100000:1.0:1761316201.160322:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307586944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316201.160456:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307586816 00000100:00100000:2.0:1761316201.160506:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1927 00000100:00100000:2.0:1761316201.160512:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307586816:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316201.160739:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000357a3bb2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307586944:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 414us (508us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316201.160751:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5276 00000100:00100000:3.0:1761316201.160853:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7587896 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307586880:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 546us (758us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316201.160938:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000030ea60cf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307586816:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 424us (742us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316201.160946:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1927 00000100:00100000:3.0:1761316201.160994:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5275 00000100:00100000:2.0:1761316201.161066:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316201.161120:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587008 00000100:00100000:0.0:1761316201.161153:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2599 00000100:00100000:0.0:1761316201.161157:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307587008:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761316201.161248:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316201.161297:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307587008:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 137us (231us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316201.161305:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2599 00010000:00010000:2.0:1761316201.181839:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316201.691686:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316201.691710:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322655872:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316201.691779:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316201.691982:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655872 00000100:00100000:1.0:1761316201.692038:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3284 00000100:00100000:1.0:1761316201.692045:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322655872:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316201.692119:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316201.692177:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004c0e1e6f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322655872:12345-0@lo:41:osp-pre-1-0.0 Request processed in 131us (399us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316201.692184:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3284 00000100:00100000:3.0:1761316201.693382:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322655872:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316202.204780:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316203.227616:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316204.251611:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316204.251668:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322656000:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316204.251676:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316204.251683:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316204.251686:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316204.251691:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4446 00000100:00080000:3.0:1761316204.251694:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316204.251700:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316204.251762:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322655936:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316204.251812:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000095d34b1 x1846869322656320/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316204.251844:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316204.251848:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316204.251850:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316204.251853:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4446 00000100:00080000:3.0:1761316204.251855:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316204.251858:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316204.251876:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009f52f74a x1846869322656384/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316204.251945:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000067f2d4b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322656192:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316204.251987:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322656064:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.252954:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316204.252998:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322656256:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.253040:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316204.253058:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322656640:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316204.253091:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316204.253129:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656000 00000100:00100000:1.0:1761316204.253168:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1928 00000100:00100000:1.0:1761316204.253184:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322656000:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.253228:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bbd41fb4 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322656000:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (277us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316204.253235:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1928 00000100:00100000:1.0:1761316204.253264:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656640 00000100:00100000:3.0:1761316204.253267:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316204.253284:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322656576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.253297:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1114 00000100:00100000:1.0:1761316204.253301:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6f0cd4a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322656640:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316204.253307:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00002000:00000024:1.0:1761316204.253322:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:3.0:1761316204.253326:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1761316204.253332:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316204.253335:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005d3ce270 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322656128:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254013:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6f0cd4a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322656640:12345-0@lo:13:osp-pre-1-1.0 Request processed in 709us (920us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316204.254023:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1114 00000100:00100000:3.0:1761316204.254032:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:2.0:1761316204.254050:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316204.254055:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000082c9e16b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322656512:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254061:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656256 00000100:00100000:3.0:1761316204.254079:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316204.254082:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656576 00000100:00100000:1.0:1761316204.254089:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322655936 00000100:00100000:1.0:1761316204.254106:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656128 00000100:00100000:3.0:1761316204.254109:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3285 00000100:00100000:3.0:1761316204.254112:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322656256:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254119:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3286 00000100:00100000:1.0:1761316204.254121:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b347652f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322655936:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316204.254140:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069b4f1dd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322656256:12345-0@lo:400:kworker/u8:2.0 Request processed in 28us (1101us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316204.254145:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3285 00000100:00100000:3.0:1761316204.254158:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3287 00000100:00100000:3.0:1761316204.254161:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322656128:12345-0@lo:400:kworker/u8:2.0 00000004:00000024:1.0:1761316204.254166:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316204.254188:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089c37b13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322656128:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316204.254193:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3287 00000100:00100000:3.0:1761316204.254211:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656192 00000100:00100000:0.0:1761316204.254245:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1683 00000100:00100000:0.0:1761316204.254249:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322656576:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316204.254275:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebc426e9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322656576:12345-0@lo:400:kworker/u8:2.0 Request processed in 26us (968us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316204.254281:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1683 00000100:00100000:0.0:1761316204.254303:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316204.254323:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656512 00000100:00100000:0.0:1761316204.254371:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656064 00000100:00100000:2.0:1761316204.254525:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5277 00000100:00100000:2.0:1761316204.254531:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322656512:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254537:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b347652f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322655936:12345-0@lo:41:osp-pre-0-1.0 Request processed in 414us (1208us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316204.254537:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5278 00000100:00100000:3.0:1761316204.254542:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322656192:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254543:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3286 00000100:00100000:1.0:1761316204.254570:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322656320:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316204.254584:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2b515d3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322656512:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (503us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316204.254586:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017a833b2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322656192:12345-0@lo:400:kworker/u8:2.0 Request processed in 43us (1318us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316204.254595:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316204.254597:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5277 00000100:00100000:3.0:1761316204.254607:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5278 00000100:00100000:1.0:1761316204.254612:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322656384:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254633:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316204.254657:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005d3ce270 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322656128:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316204.254667:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000035a0d4d2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322655936:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316204.254670:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656320 00000100:00100000:3.0:1761316204.254671:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000082c9e16b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322656512:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316204.254683:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000067f2d4b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322656192:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254686:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656384 00000100:00100000:3.0:1761316204.254696:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322656576:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254701:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2600 00000100:00100000:1.0:1761316204.254704:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322656320:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316204.254742:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2601 00000100:00100000:3.0:1761316204.254746:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322656384:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316204.254812:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316204.254846:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316204.254850:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e9cdf97f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322656320:12345-0@lo:8:kworker/u8:2.0 Request processed in 145us (256us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316204.254856:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2600 00000100:00100000:3.0:1761316204.254868:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002eb7ad1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322656384:12345-0@lo:8:kworker/u8:2.0 Request processed in 121us (234us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316204.254873:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2601 00000100:00100000:1.0:1761316204.254881:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000510104ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322656448:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316204.254897:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322656256:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254901:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316204.254911:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322656640:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316204.254918:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656448 00000100:00100000:3.0:1761316204.254920:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322656000:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254938:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5279 00000100:00100000:1.0:1761316204.254942:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322656448:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.254968:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c90fb5e5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322656448:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (67us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316204.254974:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5279 00000100:00100000:1.0:1761316204.254989:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322656704:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.255009:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316204.255011:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5280 00000100:00100000:0.0:1761316204.255016:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322656064:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.255035:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656704 00000100:00080000:2.0:1761316204.255047:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1761316204.255050:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000956eaacd pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322656064:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (746us total) trans 0 rc 0/0 00000100:00080000:2.0:1761316204.255053:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316204.255057:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5280 00000100:00100000:1.0:1761316204.255057:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1684 00000100:00100000:2.0:1761316204.255060:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322656320:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316204.255064:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b867a88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322656704:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316204.255078:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316204.255082:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316204.255086:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322656384:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316204.255090:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004654d769 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322656064:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316204.255095:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b867a88f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322656704:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (86us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316204.255100:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1684 00000100:00100000:0.0:1761316204.255103:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000510104ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322656448:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316204.255111:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322656704:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761316205.278936:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316206.284441:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316206.284494:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316206.284535:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316206.284576:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316206.284616:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587136 00000100:00100000:2.0:1761316206.284665:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587200 00000100:00100000:1.0:1761316206.284679:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5281 00000100:00100000:1.0:1761316206.284685:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a8e3ffb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307587136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316206.284726:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5282 00000100:00100000:2.0:1761316206.284740:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307587200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316206.284757:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a8e3ffb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307587136:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (262us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316206.284766:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5281 00000100:00100000:2.0:1761316206.284776:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000035a0d4d2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307587200:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 34us (199us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316206.284784:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5282 00000100:00100000:3.0:1761316206.285000:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587264 00000100:00100000:3.0:1761316206.285035:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2602 00000100:00100000:3.0:1761316206.285039:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005af8062f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307587264:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316206.285131:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316206.285176:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005af8062f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307587264:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 136us (744us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316206.285183:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2602 00000100:00100000:0.0:1761316206.285522:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587072 00000100:00100000:0.0:1761316206.285563:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1929 00000100:00100000:0.0:1761316206.285568:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307587072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316206.285612:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ecfebf31 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307587072:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 42us (1076us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316206.285619:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1929 00010000:00010000:0.0:1761316206.299480:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316206.811603:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1761316206.811654:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322656768:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316206.811725:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316206.811994:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656768 00000100:00100000:2.0:1761316206.812048:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3288 00000100:00100000:2.0:1761316206.812055:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322656768:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316206.812086:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316206.814124:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322656768:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316206.814202:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e4a4d687 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322656768:12345-0@lo:41:osp-pre-1-0.0 Request processed in 2145us (2476us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316206.814212:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3288 00010000:00010000:3.0:1761316207.323604:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316208.347897:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316209.371513:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316209.371529:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322656832:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316209.371541:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316209.371547:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316209.371550:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316209.371556:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4451 00000100:00080000:3.0:1761316209.371558:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316209.371563:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316209.371584:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000089556eb3 x1846869322657152/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316209.371616:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316209.371636:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316209.371641:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316209.371643:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316209.371647:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4451 00000100:00100000:2.0:1761316209.371649:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322657024:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316209.371649:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316209.371654:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316209.371669:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000510104ea x1846869322657216/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316209.371681:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316209.371702:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316209.371707:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322657088:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.371744:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316209.371753:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322656896:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.371765:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657408:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316209.371767:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322657152:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316209.371791:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316209.371796:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316209.371797:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316209.371818:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322657280:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.371822:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1761316209.371829:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657472:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316209.371848:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316209.371867:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322657536:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.371878:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657088 00000100:00100000:3.0:1761316209.371905:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657024 00000100:00100000:0.0:1761316209.372139:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322656960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316209.372168:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316209.372226:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656896 00000100:00100000:1.0:1761316209.372239:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000510104ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322657216:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316209.372269:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316209.372271:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657280 00000100:00100000:1.0:1761316209.372305:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656832 00000100:00100000:1.0:1761316209.372332:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1930 00000100:00100000:1.0:1761316209.372337:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322656832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.372394:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b6b4fa71 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322656832:12345-0@lo:400:kworker/u8:2.0 Request processed in 57us (793us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316209.372401:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1930 00000100:00100000:1.0:1761316209.372427:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657152 00000100:00100000:1.0:1761316209.372440:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657216 00000100:00100000:1.0:1761316209.372632:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2603 00000100:00100000:1.0:1761316209.372638:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322657152:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:1.0:1761316209.372719:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316209.372825:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657408 00000100:00100000:0.0:1761316209.372837:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322657344:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.372847:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322656960 00000100:00100000:2.0:1761316209.372863:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3289 00000100:00100000:2.0:1761316209.372870:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000086e279b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322657088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316209.372874:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316209.372886:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322657600:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316209.372921:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000086e279b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322657088:12345-0@lo:400:kworker/u8:2.0 Request processed in 49us (1174us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316209.372922:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316209.372932:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3289 00000100:00100000:2.0:1761316209.372949:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3290 00000100:00100000:2.0:1761316209.372953:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000db8d9d28 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322657408:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316209.372977:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316209.373013:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3291 00000100:00100000:0.0:1761316209.373020:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322656960:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.373028:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000db8d9d28 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322657408:12345-0@lo:41:osp-pre-0-1.0 Request processed in 73us (1232us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316209.373034:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3290 00000100:00100000:2.0:1761316209.373058:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657344 00000100:00100000:0.0:1761316209.373062:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e6dc415 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322656960:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (892us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316209.373069:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3291 00000100:00100000:0.0:1761316209.373093:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657600 00000100:00100000:0.0:1761316209.373121:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1115 00000100:00100000:0.0:1761316209.373125:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322657600:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316209.373131:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000045ef189a pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322657152:12345-0@lo:8:kworker/u8:2.0 Request processed in 490us (1331us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316209.373139:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2603 00002000:00000024:0.0:1761316209.373148:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316209.373156:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2604 00000100:00100000:1.0:1761316209.373160:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322657216:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316209.373178:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d93c9bea pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322657600:12345-0@lo:13:osp-pre-1-1.0 Request processed in 51us (259us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316209.373194:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1115 00010000:02020000:1.0:1761316209.373243:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:00010000:0.0:1761316209.373307:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316209.373332:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d4f821e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322656960:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316209.373366:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003847060c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322657600:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316209.373624:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316209.373658:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657536 00000100:00100000:1.0:1761316209.373671:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a992b434 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322657216:12345-0@lo:8:kworker/u8:2.0 Request processed in 507us (1402us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316209.373680:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2604 00000100:00100000:0.0:1761316209.373686:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1685 00000100:00100000:0.0:1761316209.373691:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322657536:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316209.373720:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316209.373724:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316209.373731:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003847060c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322657536:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (107us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316209.373738:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1685 00000100:00100000:0.0:1761316209.373766:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322657536:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316209.374240:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5283 00000100:00100000:3.0:1761316209.374249:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322657024:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.374258:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000089556eb3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322657152:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316209.374277:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316209.374280:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316209.374283:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000510104ea pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322657216:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316209.374312:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322657024:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (2617us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316209.374320:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316209.374527:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657408:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316209.374543:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322657088:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.374553:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322657024:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.374562:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f891728c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322656832:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.374612:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657472 00000100:00100000:1.0:1761316209.374641:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1686 00000100:00100000:1.0:1761316209.374645:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eebe36d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322657472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.374691:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eebe36d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322657472:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (372us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316209.374697:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1686 00000100:00100000:0.0:1761316209.374743:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5284 00000100:00100000:2.0:1761316209.374752:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5285 00000100:00100000:0.0:1761316209.374753:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000067f2d4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322656896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.374761:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322657280:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316209.374804:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000067f2d4b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322656896:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (3014us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316209.374812:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5283 00000100:00100000:0.0:1761316209.374819:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5284 00000100:00100000:2.0:1761316209.374826:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000082c9e16b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322657280:12345-0@lo:400:kworker/u8:2.0 Request processed in 65us (2979us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316209.374841:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5286 00000100:00100000:3.0:1761316209.374845:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322657344:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316209.374884:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005e264e5f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322657344:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (2015us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316209.374982:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5285 00000100:00100000:3.0:1761316209.375007:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5286 00000100:00100000:0.0:1761316209.375028:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322656896:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316209.375043:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009f52f74a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322657280:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316209.375078:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322657344:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316209.375515:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657472:0@lo:400:kworker/u8:2.0 00010000:00010000:3.0:1761316210.395512:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316211.403695:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316211.403746:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316211.403787:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316211.403862:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587392 00000100:00100000:0.0:1761316211.403916:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587456 00000100:00100000:0.0:1761316211.403936:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5287 00000100:00100000:0.0:1761316211.403942:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307587392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316211.403997:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587328 00000100:00100000:0.0:1761316211.404014:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062f7f673 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307587392:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 71us (327us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316211.404024:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5287 00000100:00100000:3.0:1761316211.404031:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1931 00000100:00100000:3.0:1761316211.404037:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307587328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316211.404044:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5288 00000100:00100000:0.0:1761316211.404051:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307587456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316211.404070:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c2e728bf pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307587328:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 31us (323us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316211.404077:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1931 00000100:00100000:0.0:1761316211.404084:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aad6d994 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307587456:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316211.404092:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5288 00000100:00100000:3.0:1761316211.407284:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316211.408065:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587520 00000100:00100000:1.0:1761316211.408801:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2605 00000100:00100000:1.0:1761316211.408809:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307587520:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:1.0:1761316211.408900:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316211.408972:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:2262:x1846869307587520:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 163us (1694us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316211.408980:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2605 00010000:00010000:3.0:1761316211.419618:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316211.931814:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316211.932856:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322657664:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316211.932930:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316211.933195:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657664 00000100:00100000:3.0:1761316211.933243:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3292 00000100:00100000:3.0:1761316211.933248:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322657664:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316211.933278:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316211.933398:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005fdf8940 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322657664:12345-0@lo:41:osp-pre-1-0.0 Request processed in 148us (466us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316211.933406:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3292 00000100:00100000:1.0:1761316211.933471:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322657664:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316212.443856:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316213.467661:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316214.491534:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322657728:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316214.491611:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761316214.491648:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316214.491713:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657728 00000100:00100000:0.0:1761316214.491764:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1116 00000100:00100000:0.0:1761316214.491769:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c731958 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322657728:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316214.491802:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316214.491850:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c731958 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322657728:12345-0@lo:13:osp-pre-1-1.0 Request processed in 80us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316214.491857:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1116 00000100:00100000:0.0:1761316214.491900:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000476db27b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322657728:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316214.491998:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316214.492012:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322657792:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316214.492054:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316214.492064:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316214.492070:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316214.492073:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316214.492078:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4456 00000100:00100000:3.0:1761316214.492080:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322657856:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316214.492082:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316214.492088:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316214.492109:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006d287d70 x1846869322658176/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316214.492116:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316214.492131:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316214.492134:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316214.492136:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1761316214.492136:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322658112:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316214.492140:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4456 00000100:00080000:1.0:1761316214.492144:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316214.492147:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316214.492159:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f5c5cc90 x1846869322658240/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316214.492169:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316214.492190:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316214.492197:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322658048:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316214.492230:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316214.492247:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657792 00000100:00100000:3.0:1761316214.492249:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316214.492257:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322658432:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492275:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658112 00000100:00100000:3.0:1761316214.492280:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316214.492299:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3293 00000100:00100000:2.0:1761316214.492303:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009f52f74a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322657792:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316214.492323:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316214.492378:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3294 00000100:00100000:3.0:1761316214.492476:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657856 00000100:00100000:2.0:1761316214.492484:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009f52f74a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322657792:12345-0@lo:41:osp-pre-0-1.0 Request processed in 178us (429us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316214.492504:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1932 00000100:00100000:3.0:1761316214.492508:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322657856:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316214.492542:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da99a8cc pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322657856:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (431us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316214.492548:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1932 00000100:00100000:0.0:1761316214.492605:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322658112:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492605:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3293 00000100:00100000:2.0:1761316214.492633:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658048 00000100:00100000:0.0:1761316214.492641:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000510104ea pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322658112:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (477us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316214.492651:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3294 00000100:00100000:2.0:1761316214.492661:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5289 00000100:00100000:2.0:1761316214.492666:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322658048:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.492685:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322658176:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492698:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000089556eb3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322658048:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (467us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316214.492705:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5289 00000100:00100000:0.0:1761316214.492721:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316214.492732:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322657920:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.492741:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322658240:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492768:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316214.492772:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316214.492789:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000255f7186 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322658304:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.492808:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658432 00000100:00100000:2.0:1761316214.492816:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316214.492836:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658176 00000100:00100000:2.0:1761316214.492841:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1761316214.492854:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658240 00000100:00100000:2.0:1761316214.492860:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657984:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.492870:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1687 00000100:00100000:0.0:1761316214.492875:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322658432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492887:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316214.492905:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322658368:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.492917:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa13e2b7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322658432:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (637us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316214.492923:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1687 00000100:00100000:2.0:1761316214.492934:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316214.492939:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2606 00000100:00100000:0.0:1761316214.492944:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322658176:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316214.492963:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657984 00000100:00100000:2.0:1761316214.492985:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3295 00000100:00100000:2.0:1761316214.492989:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ea95e939 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322657984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493021:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ea95e939 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322657984:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (135us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316214.493027:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3295 00010000:02020000:0.0:1761316214.493033:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316214.493046:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322657920 00000100:00100000:2.0:1761316214.493070:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658304 00000100:00100000:3.0:1761316214.493075:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2607 00000100:00100000:0.0:1761316214.493079:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000476db27b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322658176:12345-0@lo:8:kworker/u8:2.0 Request processed in 134us (359us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316214.493081:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322658240:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316214.493085:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2606 00000100:00100000:2.0:1761316214.493092:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658368 00000100:00100000:0.0:1761316214.493111:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5290 00000100:00100000:0.0:1761316214.493119:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322657920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493119:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5291 00000100:00100000:2.0:1761316214.493126:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322658304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.493157:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322657920:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (390us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316214.493164:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5290 00000100:00100000:2.0:1761316214.493167:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322658304:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316214.493176:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5291 00000100:00100000:0.0:1761316214.493183:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5292 00000100:00100000:0.0:1761316214.493189:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322658368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493212:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004be373e1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322657984:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761316214.493219:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316214.493227:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322657920:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493244:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000255f7186 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322658304:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316214.493246:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2415dea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322658368:12345-0@lo:400:kworker/u8:2.0 Request processed in 56us (313us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316214.493246:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f891728c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322658240:12345-0@lo:8:kworker/u8:2.0 Request processed in 165us (475us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316214.493256:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5292 00000100:00100000:3.0:1761316214.493256:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2607 00000100:00100000:2.0:1761316214.493280:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322658368:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761316214.493314:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316214.493318:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316214.493322:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006d287d70 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322658176:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761316214.493329:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316214.493331:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316214.493334:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c5cc90 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322658240:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493600:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322658432:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493611:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322658048:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493619:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000894bd804 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322657856:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493628:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd4606d5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322658112:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316214.493640:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322657792:0@lo:41:osp-pre-0-1.0 00010000:00010000:3.0:1761316215.515789:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316216.523361:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316216.524311:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587648 00000100:00100000:2.0:1761316216.524387:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316216.524431:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316216.524474:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316216.524488:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587776 00000100:00100000:2.0:1761316216.524516:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587712 00000100:00100000:0.0:1761316216.524530:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2608 00000100:00100000:0.0:1761316216.524538:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307587776:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761316216.524548:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5293 00000100:00100000:2.0:1761316216.524566:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307587712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00010000:02020000:0.0:1761316216.524648:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316216.524668:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004cf778b9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307587712:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 103us (195us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316216.524677:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5293 00000100:00100000:0.0:1761316216.524678:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307587776:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 141us (247us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316216.524683:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2608 00000100:00100000:3.0:1761316216.524761:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587584 00000100:00100000:1.0:1761316216.524860:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5294 00000100:00100000:1.0:1761316216.524866:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307587648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316216.524895:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006fc130f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307587648:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 29us (1562us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316216.525258:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5294 00000100:00100000:3.0:1761316216.525446:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1933 00000100:00100000:3.0:1761316216.525452:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307587584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316216.525543:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c1728eb1 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307587584:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 90us (1158us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316216.525566:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1933 00010000:00010000:0.0:1761316216.539564:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316217.053152:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316217.053174:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322658496:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316217.053234:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316217.053382:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658496 00000100:00100000:1.0:1761316217.053432:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3296 00000100:00100000:1.0:1761316217.053437:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322658496:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316217.053469:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316217.053986:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d4f821e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322658496:12345-0@lo:41:osp-pre-1-0.0 Request processed in 545us (751us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316217.053994:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3296 00000100:00100000:0.0:1761316217.054063:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095d34b1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322658496:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316217.563636:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316218.588399:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316219.611942:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316219.611952:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316219.611958:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316219.611970:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4462 00000100:00080000:1.0:1761316219.611974:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316219.611981:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316219.612008:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000255f7186 x1846869322658944/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316219.612024:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316219.612027:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316219.612028:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316219.612032:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4462 00000100:00080000:1.0:1761316219.612034:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316219.612037:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316219.612052:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000443cb307 x1846869322659008/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316219.612108:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322658752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612200:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612238:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322659136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612265:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612284:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:1.0:1761316219.612289:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c32f389e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322658688:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612310:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612323:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322659072:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612365:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612400:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658752 00000100:00100000:1.0:1761316219.612449:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3297 00000100:00100000:1.0:1761316219.612456:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322658752:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316219.612458:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659136 00000100:00100000:2.0:1761316219.612495:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658688 00000100:00100000:1.0:1761316219.612512:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000300d853f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322658752:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (313us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316219.612519:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659072 00000100:00100000:1.0:1761316219.612520:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3297 00000100:00100000:2.0:1761316219.612558:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5295 00000100:00100000:2.0:1761316219.612564:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322659136:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612568:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5296 00000100:00100000:1.0:1761316219.612576:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322658688:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612614:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000081cfb90a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322658688:12345-0@lo:400:kworker/u8:0.0 Request processed in 38us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.612619:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5296 00000100:00100000:1.0:1761316219.612634:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5297 00000100:00100000:1.0:1761316219.612638:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322659072:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612659:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000828d143a pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322659072:12345-0@lo:400:kworker/u8:0.0 Request processed in 21us (317us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.612663:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5297 00000100:00100000:1.0:1761316219.612700:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c32f389e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322658688:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612709:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028d524fe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322659072:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612715:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007f104f12 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322658752:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612760:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322658560:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316219.612781:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612793:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322658816:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612807:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612815:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322659200:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612833:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612848:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1761316219.612853:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b99c8f0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322658624:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612873:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612883:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322658880:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612896:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612906:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322659264:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.612920:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316219.612937:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658816 00000100:00100000:1.0:1761316219.612955:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5298 00000100:00100000:1.0:1761316219.612958:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322658816:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.613011:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028d524fe pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322658816:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (204us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.613017:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5298 00000100:00100000:1.0:1761316219.613033:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658560 00000100:00100000:1.0:1761316219.613047:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658880 00000100:00100000:1.0:1761316219.613062:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3298 00000100:00100000:1.0:1761316219.613065:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322658560:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316219.613091:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316219.613116:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3c5b3a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322658560:12345-0@lo:41:osp-pre-0-1.0 Request processed in 49us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.613121:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3298 00000100:00100000:1.0:1761316219.613132:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3299 00000100:00100000:1.0:1761316219.613136:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322658880:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.613166:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f104f12 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322658880:12345-0@lo:400:kworker/u8:0.0 Request processed in 30us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.613171:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3299 00000100:00100000:1.0:1761316219.613195:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322658880:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.613209:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009a564cc8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322658560:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316219.613216:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028173c34 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322658816:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316219.613329:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006729a9ea pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322659136:12345-0@lo:400:kworker/u8:0.0 Request processed in 764us (1063us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316219.613336:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5295 00000100:00100000:3.0:1761316219.613445:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658624 00000100:00100000:0.0:1761316219.613452:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000255f7186 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322658944:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316219.613476:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1934 00000100:00100000:3.0:1761316219.613480:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322658624:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316219.613486:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316219.613503:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322659008:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316219.613515:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc31ee8a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322658624:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (642us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316219.613521:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1934 00000100:00100000:0.0:1761316219.613523:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316219.613568:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659200 00000100:00100000:3.0:1761316219.613569:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b99c8f0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322658624:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.614900:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322659136:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.614927:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316219.614930:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659264 00000100:00100000:1.0:1761316219.614936:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322659328:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316219.614962:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322658944 00000100:00100000:3.0:1761316219.614972:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659008 00000100:00100000:3.0:1761316219.614982:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1688 00000100:00100000:1.0:1761316219.614985:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316219.614987:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1d51fbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322659200:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316219.615035:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1689 00000100:00100000:3.0:1761316219.615035:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1d51fbc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322659200:12345-0@lo:400:kworker/u8:0.0 Request processed in 47us (2201us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.615039:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c32f389e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322659264:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316219.615042:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1688 00000100:00100000:1.0:1761316219.615068:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c32f389e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322659264:12345-0@lo:400:kworker/u8:0.0 Request processed in 28us (2147us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.615073:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1689 00000100:00100000:1.0:1761316219.615085:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2609 00000100:00100000:1.0:1761316219.615088:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322658944:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316219.615203:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316219.615210:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2610 00000100:00100000:0.0:1761316219.615215:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322659008:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316219.615238:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f5c5cc90 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322658944:12345-0@lo:8:kworker/u8:0.0 Request processed in 148us (1751us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316219.615244:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2609 00000100:00100000:1.0:1761316219.615279:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659328 00010000:02020000:0.0:1761316219.615293:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316219.615303:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1117 00000100:00100000:1.0:1761316219.615306:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028173c34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322659328:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316219.615323:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd4606d5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322659008:12345-0@lo:8:kworker/u8:0.0 Request processed in 107us (1800us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316219.615328:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2610 00002000:00000024:1.0:1761316219.615328:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316219.615379:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028173c34 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322659328:12345-0@lo:13:osp-pre-1-1.0 Request processed in 71us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316219.615385:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1117 00010000:00010000:0.0:1761316219.615416:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316219.615421:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316219.615426:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316219.615431:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000255f7186 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322658944:0@lo:8:kworker/u8:0.0 00000100:00080000:1.0:1761316219.615439:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316219.615442:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316219.615443:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8373827 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322659328:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316219.615444:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322659008:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316219.615483:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000164c1058 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322659264:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316219.615493:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322659200:0@lo:400:kworker/u8:0.0 00010000:00010000:3.0:1761316220.637217:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316221.660227:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316221.688433:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316221.688486:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316221.688535:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316221.688576:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316221.688611:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588032 00000100:00100000:2.0:1761316221.688650:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2611 00000100:00100000:2.0:1761316221.688656:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307588032:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316221.688748:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316221.688804:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000894bd804 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:2262:x1846869307588032:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 146us (227us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316221.688815:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2611 00000100:00100000:1.0:1761316221.689361:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587840 00000100:00100000:1.0:1761316221.689398:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1935 00000100:00100000:1.0:1761316221.689402:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307587840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316221.689439:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b7787870 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307587840:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (1015us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316221.689445:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1935 00000100:00100000:0.0:1761316221.689557:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587904 00000100:00100000:0.0:1761316221.689597:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307587968 00000100:00100000:0.0:1761316221.689617:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5299 00000100:00100000:0.0:1761316221.689621:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307587904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316221.689654:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e8373827 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307587904:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 32us (1169us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316221.689661:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5299 00000100:00100000:0.0:1761316221.689676:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5300 00000100:00100000:0.0:1761316221.689680:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307587968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316221.689704:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b2dfaf30 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307587968:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (1170us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316221.689710:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5300 00000100:00100000:1.0:1761316222.173296:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:1.0:1761316222.173316:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005d3ce270 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322659392:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316222.173390:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316222.179639:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659392 00000100:00100000:3.0:1761316222.179711:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3300 00000100:00100000:3.0:1761316222.179718:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000255f7186 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322659392:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316222.179752:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316222.179820:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000255f7186 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322659392:12345-0@lo:41:osp-pre-1-0.0 Request processed in 101us (6430us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316222.179827:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3300 00000100:00100000:1.0:1761316222.180438:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005d3ce270 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322659392:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316222.683696:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316223.707485:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761316224.731543:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316224.731550:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316224.731554:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316224.731562:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1761316224.731565:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316224.731572:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316224.731590:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000003430de1 x1846869322659840/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316224.731606:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316224.731609:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316224.731611:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316224.731614:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4467 00000100:00080000:1.0:1761316224.731616:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316224.731619:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316224.731633:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000094d2894e x1846869322659904/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316224.731681:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322659584:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.731726:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322659712:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.731747:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316224.731963:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.731964:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322659968:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.731972:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322659456:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.731989:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322660096:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.731995:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316224.732004:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.732021:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322660224:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316224.732021:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322659520:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316224.732029:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.732378:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316224.732378:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316224.732382:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:0.0:1761316224.732392:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b99c8f0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322659648:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732416:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322659840:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732423:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316224.732446:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322660032:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732453:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.732478:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322659904:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732481:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.732522:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316224.732562:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659584 00000100:00100000:0.0:1761316224.732587:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659712 00000100:00100000:1.0:1761316224.732601:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659968 00000100:00100000:0.0:1761316224.732622:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660032 00000100:00100000:1.0:1761316224.732626:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5301 00000100:00100000:1.0:1761316224.732634:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000783e6c5c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322659584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732643:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5302 00000100:00100000:0.0:1761316224.732649:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322659712:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732689:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000783e6c5c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322659584:12345-0@lo:400:kworker/u8:2.0 Request processed in 55us (945us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316224.732697:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5301 00000100:00100000:0.0:1761316224.732704:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006d287d70 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322659712:12345-0@lo:400:kworker/u8:2.0 Request processed in 52us (741us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316224.732713:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5302 00000100:00100000:1.0:1761316224.732718:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5303 00000100:00100000:1.0:1761316224.732723:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322659968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732732:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5304 00000100:00100000:0.0:1761316224.732737:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322660032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732755:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000aa5cc910 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322659968:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (760us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316224.732763:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5303 00000100:00100000:0.0:1761316224.732774:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004654d769 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322660032:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316224.732783:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5304 00000100:00100000:1.0:1761316224.732792:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660224 00000100:00100000:1.0:1761316224.732819:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659648 00000100:00100000:0.0:1761316224.732823:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000002e606f9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322660032:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732840:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000443cb307 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322659712:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732841:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3301 00000100:00100000:1.0:1761316224.732849:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322660224:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:1.0:1761316224.732873:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316224.732897:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660096 00000100:00100000:1.0:1761316224.732906:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b0c84f13 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322660224:12345-0@lo:41:osp-pre-0-1.0 Request processed in 57us (530us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316224.732913:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3301 00000100:00100000:0.0:1761316224.732924:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659840 00000100:00100000:1.0:1761316224.732929:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3302 00000100:00100000:1.0:1761316224.732934:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322659648:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.732944:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659904 00000100:00100000:0.0:1761316224.732964:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1690 00000100:00100000:1.0:1761316224.732971:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008250083e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322659648:12345-0@lo:400:kworker/u8:2.0 Request processed in 34us (548us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316224.732972:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322660096:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316224.732983:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3302 00000100:00100000:0.0:1761316224.733011:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000164c1058 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322660096:12345-0@lo:400:kworker/u8:2.0 Request processed in 40us (982us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316224.733018:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2612 00000100:00100000:0.0:1761316224.733026:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1690 00000100:00100000:1.0:1761316224.733027:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322659840:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316224.733041:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2613 00000100:00100000:0.0:1761316224.733046:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322659904:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316224.733478:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322659776:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.733517:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316224.733530:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000949f9e44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322660160:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.733547:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:3.0:1761316224.733570:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316224.733585:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659456 00000100:00100000:3.0:1761316224.733607:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1936 00000100:00100000:3.0:1761316224.733610:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322659456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.733654:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e321a9ed pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322659456:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (1651us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316224.733659:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1936 00010000:02020000:0.0:1761316224.733681:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316224.733684:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659776 00000100:00100000:3.0:1761316224.733710:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3303 00000100:00100000:3.0:1761316224.733714:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322659776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.733725:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a425d4ca pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322659904:12345-0@lo:8:kworker/u8:2.0 Request processed in 676us (1203us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316224.733734:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2613 00000100:00100000:3.0:1761316224.733750:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a564cc8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322659776:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (231us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316224.733751:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660160 00000100:00100000:3.0:1761316224.733758:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3303 00000100:00100000:0.0:1761316224.733774:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1691 00000100:00100000:0.0:1761316224.733778:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095d34b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322660160:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.733786:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322659520 00000100:00100000:3.0:1761316224.733817:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1118 00000100:00100000:0.0:1761316224.733820:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095d34b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322660160:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (269us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316224.733825:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322659520:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316224.733832:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1691 00002000:00000024:3.0:1761316224.733848:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:0.0:1761316224.733879:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316224.733883:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316224.733887:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000094d2894e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322659904:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316224.734008:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3ce270 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322659520:12345-0@lo:13:osp-pre-1-1.0 Request processed in 184us (1632us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316224.734016:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1118 00000100:00100000:3.0:1761316224.734053:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a0ce68db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322659456:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.734068:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001b3dd283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322659520:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316224.734076:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a2322d75 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322659776:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316224.734085:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000949f9e44 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322660160:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316224.734141:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322660224:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316224.734156:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006b9872de pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322659584:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316224.734166:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000046bac38f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322659968:0@lo:400:kworker/u8:2.0 00010000:02020000:1.0:1761316224.737018:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316224.737800:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c3799918 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322659840:12345-0@lo:8:kworker/u8:2.0 Request processed in 4773us (5349us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316224.737813:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2612 00000100:00080000:2.0:1761316224.738179:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316224.738184:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316224.738188:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000003430de1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322659840:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316224.739519:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b99c8f0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322659648:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316224.739546:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322660096:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316225.756256:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316226.764423:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316226.764478:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316226.764516:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316226.764550:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316226.764594:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588224 00000100:00100000:0.0:1761316226.764598:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588096 00000100:00100000:0.0:1761316226.764650:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1937 00000100:00100000:1.0:1761316226.764651:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588160 00000100:00100000:0.0:1761316226.764659:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307588096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316226.764683:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5305 00000100:00100000:3.0:1761316226.764695:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307588224:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316226.764747:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001e39bf5c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307588096:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 87us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316226.764756:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1937 00000100:00100000:3.0:1761316226.764771:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000388257a5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307588224:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 76us (253us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316226.765282:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588288 00000100:00100000:2.0:1761316226.765314:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2614 00000100:00100000:2.0:1761316226.765320:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307588288:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316226.765462:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316226.765522:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002aaef6f9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307588288:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 201us (1108us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316226.765530:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2614 00000100:00100000:1.0:1761316226.766194:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5306 00000100:00100000:1.0:1761316226.766209:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307588160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316226.766248:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ff653b22 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307588160:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 41us (1699us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316226.766254:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5306 00000100:00100000:3.0:1761316226.767153:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5305 00010000:00010000:2.0:1761316226.779593:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316227.293198:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316227.293223:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322660288:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316227.293293:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316227.293463:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660288 00000100:00100000:3.0:1761316227.293535:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3304 00000100:00100000:3.0:1761316227.293541:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322660288:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316227.293574:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316227.293629:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000949f9e44 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322660288:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (336us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316227.293636:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3304 00000100:00100000:0.0:1761316227.293766:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322660288:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316227.807634:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316228.828069:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316229.853562:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322660352:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316229.853592:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316229.853639:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316229.853743:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660352 00000100:00100000:0.0:1761316229.853793:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1119 00000100:00100000:0.0:1761316229.853800:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322660352:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316229.853826:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316229.853942:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322660352:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316229.854754:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008de39ee1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322660352:12345-0@lo:13:osp-pre-1-1.0 Request processed in 952us (1114us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316229.854766:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1119 00000100:00080000:1.0:1761316229.854773:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316229.854779:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316229.854783:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316229.854788:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1761316229.854791:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316229.854796:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316229.854812:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322660416:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.854820:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000036dbb695 x1846869322660800/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316229.854825:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322660480:0@lo:400:kworker/u8:2.0 00000100:00080000:1.0:1761316229.854843:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316229.854847:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316229.854848:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1761316229.854854:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316229.854855:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4472 00000100:00080000:1.0:1761316229.854859:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316229.854862:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316229.854874:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316229.854877:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322660672:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.854895:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e376567a x1846869322660864/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316229.854908:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322660736:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316229.854915:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316229.854945:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316229.854947:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661056:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.854970:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660672 00000100:00100000:0.0:1761316229.854997:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316229.855006:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660736 00000100:00100000:3.0:1761316229.855019:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660480 00000100:00100000:1.0:1761316229.855041:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3305 00000100:00100000:0.0:1761316229.855045:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322660544:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855050:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322660672:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855060:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5307 00000100:00100000:3.0:1761316229.855067:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322660480:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855074:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316229.855088:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322660928:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855108:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000094d2894e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322660672:12345-0@lo:400:kworker/u8:2.0 Request processed in 58us (194us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855119:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3305 00000100:00100000:0.0:1761316229.855123:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316229.855132:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322660480:12345-0@lo:400:kworker/u8:2.0 Request processed in 64us (259us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855139:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660544 00000100:00100000:3.0:1761316229.855141:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5307 00000100:00100000:0.0:1761316229.855147:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:0.0:1761316229.855156:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322660608:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855158:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3306 00000100:00100000:1.0:1761316229.855165:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322660736:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316229.855177:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660928 00000100:00100000:0.0:1761316229.855192:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000004:00000024:1.0:1761316229.855193:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316229.855212:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660608 00000100:00100000:0.0:1761316229.855218:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322660992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855229:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a2322d75 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+7:3354:x1846869322660736:12345-0@lo:41:osp-pre-0-1.0 Request processed in 64us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855237:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3306 00000100:00100000:3.0:1761316229.855240:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5308 00000100:00100000:3.0:1761316229.855246:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322660928:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855253:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316229.855254:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3307 00000100:00100000:1.0:1761316229.855261:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322660544:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855294:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000719d773a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322660672:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855298:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000002e606f9 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322660928:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855300:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000046bac38f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322660544:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (227us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316229.855314:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5308 00000100:00100000:1.0:1761316229.855315:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3307 00000100:00100000:0.0:1761316229.855340:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660992 00000100:00100000:3.0:1761316229.855364:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5309 00000100:00100000:3.0:1761316229.855370:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322660608:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855381:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5310 00000100:00100000:1.0:1761316229.855388:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322660992:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855404:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322660608:12345-0@lo:400:kworker/u8:2.0 Request processed in 32us (213us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316229.855410:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5309 00000100:00100000:1.0:1761316229.855440:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322660992:12345-0@lo:400:kworker/u8:2.0 Request processed in 51us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855447:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5310 00000100:00100000:0.0:1761316229.855470:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660416 00000100:00100000:3.0:1761316229.855470:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e53cb60 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322660480:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855489:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fa13e2b7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322660736:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316229.855492:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322660800:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855507:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1938 00000100:00100000:0.0:1761316229.855514:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322660416:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855527:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bb94dc5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322660608:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855528:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316229.855544:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000062800435 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322660992:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855553:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322660864:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855559:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cbfcf664 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322660544:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855565:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000926d4b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322660416:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (713us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316229.855572:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1938 00000100:00100000:3.0:1761316229.855574:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001198e496 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322660928:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855583:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316229.855625:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004870ba5e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322660416:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855630:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661056 00000100:00100000:3.0:1761316229.855632:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660800 00000100:00100000:3.0:1761316229.855660:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322660864 00000100:00100000:1.0:1761316229.855677:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1692 00000100:00100000:1.0:1761316229.855682:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661056:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316229.855687:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2615 00000100:00100000:0.0:1761316229.855697:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2616 00000100:00100000:3.0:1761316229.855697:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322660800:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316229.855704:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322660864:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316229.855721:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000443cb307 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661056:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (725us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316229.855728:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1692 00000100:00100000:1.0:1761316229.855763:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000008d853e9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661056:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761316229.856184:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316229.856188:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316229.856242:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c96a4a50 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322660864:12345-0@lo:8:kworker/u8:2.0 Request processed in 538us (661us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316229.856246:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034b089b1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322660800:12345-0@lo:8:kworker/u8:2.0 Request processed in 550us (718us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316229.856256:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2616 00000100:00100000:3.0:1761316229.856261:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2615 00000100:00080000:1.0:1761316229.858198:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316229.858207:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316229.858213:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000036dbb695 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322660800:0@lo:8:kworker/u8:2.0 00000100:00080000:1.0:1761316229.858224:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316229.858227:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316229.858229:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e376567a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322660864:0@lo:8:kworker/u8:2.0 00010000:00010000:3.0:1761316230.879128:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316231.884738:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316231.884794:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316231.885412:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588416 00000100:00100000:2.0:1761316231.885495:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5311 00000100:00100000:2.0:1761316231.885503:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307588416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316231.885591:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b10b1dae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307588416:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 88us (861us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316231.885599:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5311 00000100:00100000:0.0:1761316231.886362:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588352 00000100:00100000:0.0:1761316231.886547:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1939 00000100:00100000:0.0:1761316231.886553:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307588352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316231.886606:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d0d33188 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307588352:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 53us (1813us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316231.886612:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1939 00000100:00100000:1.0:1761316231.886889:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316231.886929:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316231.887038:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588480 00000100:00100000:0.0:1761316231.887043:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588544 00000100:00100000:3.0:1761316231.887075:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5312 00000100:00100000:0.0:1761316231.887080:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2617 00000100:00100000:3.0:1761316231.887083:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc6abe87 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307588480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316231.887090:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307588544:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:3.0:1761316231.887122:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc6abe87 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307588480:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 40us (234us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316231.887129:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5312 00010000:02020000:0.0:1761316231.887181:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316231.887209:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007f54c1b9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307588544:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 121us (281us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316231.887216:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2617 00010000:00010000:0.0:1761316231.899837:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316232.411979:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316232.412001:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661120:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316232.412063:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316232.412828:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661120 00000100:00100000:2.0:1761316232.412886:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3308 00000100:00100000:2.0:1761316232.412891:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661120:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316232.412926:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316232.413091:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661120:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316232.413149:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000978c22f3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661120:12345-0@lo:41:osp-pre-1-0.0 Request processed in 255us (1084us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316232.413157:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3308 00010000:00010000:0.0:1761316232.923537:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316233.947757:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316234.971514:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316234.971554:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661248:0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316234.971565:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3ecfafc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322661184:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316234.971577:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316234.971583:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069f2cc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322661376:0@lo:400:kworker/u8:0.0 00000100:00080000:1.0:1761316234.971584:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316234.971589:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316234.971599:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4477 00000100:00080000:1.0:1761316234.971602:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1761316234.971609:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316234.971632:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001a8c16f2 x1846869322661632/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761316234.971649:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1761316234.971653:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1761316234.971655:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1761316234.971664:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4477 00000100:00100000:0.0:1761316234.971669:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316234.971671:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761316234.971678:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316234.971678:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:1.0:1761316234.971683:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316234.971698:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005c22bba0 x1846869322661696/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316234.971717:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b5a51ecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322661504:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316234.971756:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661248 00000100:00100000:0.0:1761316234.971815:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3309 00000100:00100000:0.0:1761316234.971821:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322661248:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316234.971863:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316234.971903:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d75a9b1 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322661248:12345-0@lo:41:osp-pre-0-1.0 Request processed in 81us (248us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316234.971910:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3309 00000100:00100000:0.0:1761316234.972018:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322661312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316234.972046:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316234.972545:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001a8c16f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322661632:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316234.972578:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316234.972597:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c22bba0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322661696:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316234.972617:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316234.972645:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661632 00000100:00100000:1.0:1761316234.972665:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661696 00000100:00100000:1.0:1761316234.972676:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2618 00000100:00100000:1.0:1761316234.972680:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000033e2b8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322661632:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973119:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322661760:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973147:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316234.973159:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316234.973164:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322661952:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973179:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316234.973204:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661376 00000100:00100000:2.0:1761316234.973228:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661760 00000100:00100000:2.0:1761316234.973241:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5313 00000100:00100000:2.0:1761316234.973245:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a1daabeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322661376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973277:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a1daabeb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322661376:12345-0@lo:400:kworker/u8:0.0 Request processed in 32us (1618us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316234.973283:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5313 00000100:00100000:2.0:1761316234.973292:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5314 00000100:00100000:2.0:1761316234.973296:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d70cea3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322661760:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973312:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d70cea3d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322661760:12345-0@lo:400:kworker/u8:0.0 Request processed in 16us (166us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316234.973317:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5314 00000100:00100000:2.0:1761316234.973334:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316234.973379:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661248:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316234.973433:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661888:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973460:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316234.973516:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [2->3] 00000100:00100000:2.0:1761316234.973530:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000596f0dd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661440:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973557:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316234.973562:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661504 00000100:00100000:2.0:1761316234.973577:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322661824:0@lo:400:kworker/u8:0.0 00010000:02020000:1.0:1761316234.973594:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316234.973603:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316234.973630:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661824 00000100:00100000:1.0:1761316234.973660:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000033e2b8a8 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322661632:12345-0@lo:8:kworker/u8:0.0 Request processed in 975us (1078us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316234.973660:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661952 00000100:00100000:3.0:1761316234.973675:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5315 00000100:00100000:2.0:1761316234.973693:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661888 00000100:00100000:3.0:1761316234.973697:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e3bfc07b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322661504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316234.973698:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2618 00000100:00100000:2.0:1761316234.973712:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1693 00000100:00100000:2.0:1761316234.973719:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322661952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316234.973725:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1694 00000100:00100000:1.0:1761316234.973732:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322661888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.973780:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322661952:12345-0@lo:400:kworker/u8:0.0 Request processed in 61us (599us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316234.973789:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1693 00000100:00100000:1.0:1761316234.973795:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322661888:12345-0@lo:400:kworker/u8:0.0 Request processed in 59us (332us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316234.973804:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1694 00000100:00100000:3.0:1761316234.973816:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e3bfc07b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3356:x1846869322661504:12345-0@lo:400:kworker/u8:0.0 Request processed in 122us (473us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316234.973817:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2619 00000100:00100000:2.0:1761316234.973832:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a014551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322661696:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316234.973850:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5315 00000100:00080000:1.0:1761316234.973865:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761316234.973873:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316234.973879:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5316 00000100:00100000:1.0:1761316234.973880:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001a8c16f2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322661632:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316234.973892:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008a0e4173 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322661824:12345-0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761316234.973923:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316234.973941:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661440 00000100:00100000:2.0:1761316234.973954:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a014551 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322661696:12345-0@lo:8:kworker/u8:0.0 Request processed in 129us (1336us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316234.973962:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2619 00000100:00100000:1.0:1761316234.973975:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3310 00000100:00100000:1.0:1761316234.973981:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000059f3b7c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322661440:12345-0@lo:400:kworker/u8:0.0 00000100:00080000:2.0:1761316234.973989:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316234.973994:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316234.973997:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c22bba0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322661696:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316234.974029:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000099aed9bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322661952:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316234.974034:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000059f3b7c3 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322661440:12345-0@lo:400:kworker/u8:0.0 Request processed in 52us (476us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316234.974042:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3310 00000100:00100000:2.0:1761316234.974047:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000b653f5d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322661760:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316234.974051:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000328f1831 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661568:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.974063:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069f2cc23 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322661376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316234.974084:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316234.974100:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000596f0dd4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661440:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316234.974107:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661184 00000100:00100000:2.0:1761316234.974117:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000024a9e32e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322661888:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.974132:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b5a51ecd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0001_UUID:3356:1846869322661504:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316234.974134:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661568 00000100:00100000:1.0:1761316234.974144:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1940 00000100:00100000:1.0:1761316234.974149:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322661184:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316234.974189:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f2c78b25 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322661184:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (2532us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316234.974195:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1940 00000100:00100000:1.0:1761316234.974222:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3311 00000100:00100000:1.0:1761316234.974226:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.974245:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322661312 00000100:00100000:2.0:1761316234.974273:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1120 00000100:00100000:2.0:1761316234.974277:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322661312:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316234.974295:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:1.0:1761316234.974585:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d029cd1d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322661568:12345-0@lo:400:kworker/u8:0.0 Request processed in 354us (499us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316234.974593:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008a0e4173 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322661824:12345-0@lo:400:kworker/u8:0.0 Request processed in 697us (984us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316234.974601:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3311 00000100:00100000:3.0:1761316234.974610:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5316 00000100:00100000:3.0:1761316234.974669:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3ecfafc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322661184:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316234.974728:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000000352602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322661824:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316234.975923:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee723cf2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322661312:12345-0@lo:13:osp-pre-1-1.0 Request processed in 1640us (3872us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316234.975935:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1120 00000100:00100000:0.0:1761316234.976052:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000054e14ecf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322661312:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316234.976066:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000328f1831 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322661568:0@lo:400:kworker/u8:0.0 00010000:00010000:0.0:1761316235.995600:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316236.999698:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316236.999752:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316236.999804:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316236.999879:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588672 00000100:00100000:2.0:1761316236.999934:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588736 00000100:00100000:2.0:1761316236.999954:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5317 00000100:00100000:2.0:1761316236.999961:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005296ee71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307588672:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316237.000033:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005296ee71 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2263:x1846869307588672:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 72us (344us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316237.000041:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5317 00000100:00100000:2.0:1761316237.000055:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5318 00000100:00100000:2.0:1761316237.000058:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009a90ace4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307588736:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316237.000076:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009a90ace4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307588736:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 17us (275us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316237.000081:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5318 00000100:00100000:1.0:1761316237.000506:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588608 00000100:00100000:2.0:1761316237.000728:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1941 00000100:00100000:2.0:1761316237.000733:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307588608:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316237.000756:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004feeb70d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307588608:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 22us (1005us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316237.000763:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1941 00000100:00100000:0.0:1761316237.007799:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316237.008605:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588800 00000100:00100000:1.0:1761316237.008656:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2620 00000100:00100000:1.0:1761316237.008662:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000beffc925 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307588800:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:1.0:1761316237.008758:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316237.008832:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000beffc925 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307588800:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 168us (1041us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316237.008841:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2620 00010000:00010000:3.0:1761316237.020078:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316237.536242:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322662016:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316237.536313:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316237.536516:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662016 00000100:00100000:1.0:1761316237.536573:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3312 00000100:00100000:1.0:1761316237.536579:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000920ff855 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322662016:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316237.536609:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316237.538080:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000920ff855 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322662016:12345-0@lo:41:osp-pre-1-0.0 Request processed in 1496us (1764us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316237.538092:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3312 00000100:00100000:2.0:1761316237.538314:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322662016:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316238.044396:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316239.067541:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316240.092786:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:2.0:1761316240.092803:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cd3bcd15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322662080:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316240.092871:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316240.093529:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316240.093542:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322662144:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316240.093572:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316240.093653:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662144 00000100:00100000:0.0:1761316240.093709:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1121 00000100:00100000:0.0:1761316240.093715:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a7398 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322662144:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316240.093742:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316240.093780:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a7398 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322662144:12345-0@lo:13:osp-pre-1-1.0 Request processed in 64us (208us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316240.093788:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1121 00000100:00100000:1.0:1761316240.093863:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662080 00000100:00100000:1.0:1761316240.093898:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3313 00000100:00100000:1.0:1761316240.093904:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000040d7daf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322662080:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316240.093917:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000fa3b911 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322662144:0@lo:13:osp-pre-1-1.0 00000004:00000024:1.0:1761316240.094027:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00010000:00010000:3.0:1761316240.094035:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316240.094072:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000040d7daf8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322662080:12345-0@lo:41:osp-pre-0-1.0 Request processed in 167us (1201us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316240.094080:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3313 00000100:00100000:3.0:1761316240.094092:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cd3bcd15 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322662080:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316240.094497:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ba8be421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322662208:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316240.094536:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316240.094757:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662208 00000100:00100000:2.0:1761316240.094792:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1942 00000100:00100000:2.0:1761316240.094797:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322662208:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.094897:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322662272:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.094933:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316240.094967:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662272 00000100:00100000:3.0:1761316240.094994:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5319 00000100:00100000:3.0:1761316240.094998:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322662272:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.095029:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cd3bcd15 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322662272:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (96us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316240.095035:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5319 00000100:00100000:3.0:1761316240.095066:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322662272:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.095124:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000064a78278 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322662208:12345-0@lo:400:kworker/u8:2.0 Request processed in 327us (589us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316240.095131:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1942 00000100:00080000:0.0:1761316240.095606:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316240.095611:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316240.095614:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316240.095619:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4482 00000100:00080000:0.0:1761316240.095622:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316240.095626:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316240.095644:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000067d9ad19 x1846869322662464/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316240.095658:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316240.095661:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316240.095662:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316240.095665:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4482 00000100:00080000:0.0:1761316240.095667:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316240.095669:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316240.095676:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b6424f0 x1846869322662528/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316240.095782:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ba8be421 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322662208:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.095792:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000067d9ad19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322662464:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316240.095827:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316240.095853:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b6424f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322662528:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316240.095872:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322662592:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.095876:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316240.095908:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322662336:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316240.095912:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316240.096053:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662592 00000100:00100000:3.0:1761316240.096080:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5320 00000100:00100000:3.0:1761316240.096084:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a8c16f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322662592:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316240.096900:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316240.096909:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322662656:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.097657:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316240.097777:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a8c16f2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3353:x1846869322662592:12345-0@lo:400:kworker/u8:2.0 Request processed in 1691us (1867us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316240.097790:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5320 00000100:00100000:1.0:1761316240.097867:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662464 00000100:00100000:1.0:1761316240.097899:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662528 00000100:00100000:1.0:1761316240.097914:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2621 00000100:00100000:1.0:1761316240.097919:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000fa3b911 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322662464:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316240.097950:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322662720:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.097981:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316240.098000:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316240.098008:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322662400:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.098029:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316240.098070:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662720 00000100:00100000:0.0:1761316240.098101:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1695 00000100:00100000:0.0:1761316240.098105:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322662720:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316240.098156:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6650b19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322662720:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316240.098163:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1695 00000100:00100000:0.0:1761316240.098173:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2622 00000100:00100000:0.0:1761316240.098176:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322662528:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:0.0:1761316240.098264:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316240.098297:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008070be5c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322662528:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (2424us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316240.098303:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2622 00000100:00080000:0.0:1761316240.098336:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316240.098340:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316240.098364:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b6424f0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322662528:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316240.098405:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000013d099c5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322662720:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316240.098448:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316240.098845:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662336 00000100:00100000:3.0:1761316240.098872:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5321 00000100:00100000:3.0:1761316240.098875:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322662336:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.098911:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003fc163cb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322662336:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (1258us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316240.098917:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5321 00000100:00100000:3.0:1761316240.098966:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662400 00000100:00100000:3.0:1761316240.098985:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3314 00000100:00100000:3.0:1761316240.098988:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322662400:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.099010:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000018d8e118 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322662400:12345-0@lo:400:kworker/u8:2.0 Request processed in 21us (981us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316240.099014:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3314 00000100:00100000:3.0:1761316240.099044:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d332521 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322662400:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316240.099052:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000008859f1fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322662336:0@lo:400:kworker/u8:2.0 00010000:02020000:1.0:1761316240.099468:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316240.099509:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000fa3b911 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322662464:12345-0@lo:8:kworker/u8:2.0 Request processed in 1590us (3682us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316240.099517:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2621 00000100:00080000:3.0:1761316240.101189:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316240.101197:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316240.101203:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000067d9ad19 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322662464:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316240.105733:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000102983aa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-OST0001_UUID:3353:1846869322662592:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.106192:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662656 00000100:00100000:2.0:1761316240.106251:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5322 00000100:00100000:2.0:1761316240.106258:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322662656:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316240.106993:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000013d099c5 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322662656:12345-0@lo:400:kworker/u8:2.0 Request processed in 734us (8543us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316240.107003:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5322 00000100:00100000:0.0:1761316240.107907:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d9493d69 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322662656:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316241.115560:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316242.120094:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316242.120146:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316242.120188:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316242.120295:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588864 00000100:00100000:1.0:1761316242.120482:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588992 00000100:00100000:1.0:1761316242.120515:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307588928 00000100:00100000:1.0:1761316242.120532:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5323 00000100:00100000:1.0:1761316242.120538:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307588992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316242.120602:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000812c1f62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2263:x1846869307588992:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 63us (457us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316242.120609:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5323 00000100:00100000:1.0:1761316242.120622:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5324 00000100:00100000:1.0:1761316242.120625:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307588928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316242.120641:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000166a802b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307588928:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 15us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316242.120646:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5324 00000100:00100000:3.0:1761316242.121856:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1943 00000100:00100000:3.0:1761316242.121865:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307588864:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316242.121930:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f3859083 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2263:x1846869307588864:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 66us (1845us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316242.121938:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1943 00000100:00100000:0.0:1761316242.122409:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316242.123329:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589056 00000100:00100000:2.0:1761316242.123442:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2623 00000100:00100000:2.0:1761316242.123448:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589056:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:2.0:1761316242.123542:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316242.123590:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9493d69 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589056:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 142us (1182us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316242.123596:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2623 00010000:00010000:0.0:1761316242.139636:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316242.651781:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322662784:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316242.651850:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316242.651967:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662784 00000100:00100000:3.0:1761316242.652015:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3315 00000100:00100000:3.0:1761316242.652021:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322662784:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316242.652050:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316242.652095:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd68be7c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322662784:12345-0@lo:41:osp-pre-1-0.0 Request processed in 74us (247us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316242.652101:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3315 00000100:00100000:2.0:1761316242.652178:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000097937f1c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322662784:0@lo:41:osp-pre-1-0.0 00010000:00010000:1.0:1761316243.163572:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316244.187613:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316245.211515:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322662848:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316245.211522:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316245.211614:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:3.0:1761316245.211614:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316245.211620:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316245.211624:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316245.211631:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4487 00000100:00080000:3.0:1761316245.211634:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316245.211640:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316245.211658:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002e2054a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322662912:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316245.211662:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a68dea11 x1846869322663296/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316245.211678:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316245.211681:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316245.211683:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316245.211687:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4487 00000100:00080000:3.0:1761316245.211690:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316245.211692:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316245.211699:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.211705:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004c227bf8 x1846869322663360/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316245.211714:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322663104:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316245.211747:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322663168:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.211757:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.211757:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322663040:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.211788:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322663488:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316245.211801:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316245.211819:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662912 00000100:00100000:1.0:1761316245.211867:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1122 00000100:00100000:1.0:1761316245.211881:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322662912:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:1.0:1761316245.211916:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316245.212020:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.212046:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322663424:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316245.212074:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.212108:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322663296:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316245.212137:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316245.212139:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662848 00000100:00100000:3.0:1761316245.212154:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322663360:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212178:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663104 00000100:00100000:3.0:1761316245.212179:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316245.212199:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000087c09a1f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322662912:12345-0@lo:13:osp-pre-1-1.0 Request processed in 316us (500us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316245.212205:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3316 00000100:00100000:1.0:1761316245.212213:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1122 00000100:00100000:2.0:1761316245.212214:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000590ef1c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322662848:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:3.0:1761316245.212223:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663040 00000004:00000024:2.0:1761316245.212240:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316245.212256:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663488 00000100:00100000:1.0:1761316245.212271:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663424 00000100:00100000:2.0:1761316245.212277:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000590ef1c4 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322662848:12345-0@lo:41:osp-pre-0-1.0 Request processed in 64us (666us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316245.212286:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3316 00000100:00100000:3.0:1761316245.212286:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5325 00000100:00100000:3.0:1761316245.212294:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322663040:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212308:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3317 00000100:00100000:2.0:1761316245.212314:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322663104:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316245.212318:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5326 00000100:00100000:1.0:1761316245.212325:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322663488:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316245.212376:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000041f26507 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322663040:12345-0@lo:400:kworker/u8:2.0 Request processed in 48us (541us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316245.212383:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5327 00000100:00100000:3.0:1761316245.212391:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5325 00000100:00100000:0.0:1761316245.212392:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322663424:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212398:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006386da6d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322663104:12345-0@lo:400:kworker/u8:2.0 Request processed in 83us (645us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316245.212406:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3317 00000100:00100000:3.0:1761316245.212432:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663296 00000100:00100000:0.0:1761316245.212437:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000851117f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322663424:12345-0@lo:400:kworker/u8:2.0 Request processed in 46us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316245.212445:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5327 00000100:00100000:2.0:1761316245.212449:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322662976:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316245.212456:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663360 00000100:00100000:0.0:1761316245.212474:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316245.212487:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f9cd6c46 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322663488:12345-0@lo:400:kworker/u8:2.0 Request processed in 158us (463us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316245.212487:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2624 00000100:00100000:2.0:1761316245.212497:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.212497:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322663296:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316245.212499:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5326 00000100:00100000:0.0:1761316245.212519:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322662848:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316245.212522:0:6256:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663168 00000100:00100000:2.0:1761316245.212522:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663232:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316245.212545:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002e2054a2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322662912:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316245.212553:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316245.212556:0:6256:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5328 00000100:00100000:1.0:1761316245.212561:0:6256:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322663168:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212576:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316245.212585:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663552:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761316245.212589:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316245.212608:0:6256:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000102983aa pname:cluuid+ref:pid:xid:nid:opc:job mdt00_002:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322663168:12345-0@lo:400:kworker/u8:2.0 Request processed in 45us (136us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316245.212615:0:6256:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5328 00000100:00100000:0.0:1761316245.212621:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322663168:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212621:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316245.212642:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322663296:12345-0@lo:8:kworker/u8:2.0 Request processed in 146us (504us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316245.212651:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2624 00000100:00100000:2.0:1761316245.212664:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322662976 00000100:00100000:3.0:1761316245.212669:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663552 00000100:00100000:1.0:1761316245.212673:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2625 00000100:00100000:1.0:1761316245.212679:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322663360:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212693:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1944 00000100:00100000:2.0:1761316245.212699:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322662976:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316245.212703:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1696 00000100:00100000:3.0:1761316245.212709:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663552:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212738:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000710ad7f5 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322662976:12345-0@lo:400:kworker/u8:2.0 Request processed in 39us (247us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316245.212747:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1944 00000100:00100000:3.0:1761316245.212749:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002e2054a2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663552:12345-0@lo:400:kworker/u8:2.0 Request processed in 36us (128us total) trans 0 rc 0/0 00010000:02020000:1.0:1761316245.212758:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316245.212759:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1696 00000100:00100000:2.0:1761316245.212782:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663232 00000100:00100000:1.0:1761316245.212797:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322663360:12345-0@lo:8:kworker/u8:2.0 Request processed in 116us (616us total) trans 0 rc -19/-19 00000100:00080000:3.0:1761316245.212803:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1761316245.212805:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2625 00000100:00080000:3.0:1761316245.212808:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316245.212809:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3318 00000100:00100000:3.0:1761316245.212815:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322663296:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212817:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663232:12345-0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316245.212830:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316245.212834:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316245.212838:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322663360:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316245.212852:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afbb551f pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663232:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316245.212858:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3318 00000100:00100000:1.0:1761316245.212869:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000034aae76b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322663040:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316245.212890:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322663424:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316245.214735:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322663488:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316245.214751:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322663104:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.214755:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663552:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.214774:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322662976:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316245.214786:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663232:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761316246.236016:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316247.241418:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316247.241509:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589312 00000100:00100000:2.0:1761316247.241560:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2626 00000100:00100000:2.0:1761316247.241567:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589312:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316247.241660:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316247.241729:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000627dfbad pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589312:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 161us (317us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316247.241737:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2626 00000100:00100000:0.0:1761316247.248937:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316247.248981:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316247.249022:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316247.249056:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589184 00000100:00100000:1.0:1761316247.249106:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589248 00000100:00100000:1.0:1761316247.249129:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5329 00000100:00100000:1.0:1761316247.249136:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004b6424f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307589184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316247.249146:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5330 00000100:00100000:3.0:1761316247.249153:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307589248:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316247.249289:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589120 00000100:00100000:2.0:1761316247.249652:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1945 00000100:00100000:1.0:1761316247.249708:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004b6424f0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307589184:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 414us (569us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316247.249717:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5329 00000100:00100000:2.0:1761316247.250514:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316247.250557:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b4668cac pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589120:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 899us (1630us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316247.250564:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1945 00000100:00100000:3.0:1761316247.250577:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d1a5e34c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307589248:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 397us (530us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316247.250587:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5330 00010000:00010000:2.0:1761316247.259541:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316247.771599:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663616:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316247.771668:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316247.771772:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663616 00000100:00100000:0.0:1761316247.771829:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3319 00000100:00100000:0.0:1761316247.771836:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663616:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:0.0:1761316247.771867:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:0.0:1761316247.772554:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000054e14ecf pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322663616:12345-0@lo:41:osp-pre-1-0.0 Request processed in 716us (885us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316247.772561:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3319 00000100:00100000:2.0:1761316247.775926:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322663616:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316248.283593:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1761316249.307539:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316250.331510:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322663680:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316250.331537:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316250.331542:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316250.331545:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316250.331550:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4492 00000100:00080000:3.0:1761316250.331553:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316250.331559:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:1.0:1761316250.331577:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316250.331578:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000001670e17f x1846869322664000/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316250.331596:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316250.331599:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316250.331600:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1761316250.331605:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322663936:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316250.331605:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4492 00000100:00080000:3.0:1761316250.331609:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316250.331613:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316250.331626:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f6aa85f4 x1846869322664064/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316250.331636:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.331656:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761316250.331664:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322663744:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316250.331680:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322664000:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316250.331688:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.331703:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322664128:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.331729:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316250.331739:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.331745:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322664320:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316250.331765:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322664064:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316250.331772:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316250.331796:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.331811:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663936 00000100:00100000:3.0:1761316250.331851:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663744 00000100:00100000:1.0:1761316250.331853:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3320 00000100:00100000:1.0:1761316250.331859:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322663936:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316250.331885:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664128 00000100:00100000:1.0:1761316250.331902:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b596d59b pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322663936:12345-0@lo:400:kworker/u8:0.0 Request processed in 41us (268us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316250.331910:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3320 00000100:00100000:3.0:1761316250.331926:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5331 00000100:00100000:3.0:1761316250.331932:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000328f1831 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322663744:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.331935:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664000 00000100:00100000:1.0:1761316250.331954:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664320 00000100:00100000:3.0:1761316250.331973:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000328f1831 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322663744:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316250.331975:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664064 00000100:00100000:3.0:1761316250.331982:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5331 00000100:00100000:1.0:1761316250.331991:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1697 00000100:00100000:1.0:1761316250.331996:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d3ecfafc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322664320:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316250.332000:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5332 00000100:00100000:3.0:1761316250.332006:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000000352602 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322664128:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.332029:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d3ecfafc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322664320:12345-0@lo:400:kworker/u8:0.0 Request processed in 31us (257us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316250.332036:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1697 00000100:00100000:3.0:1761316250.332043:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000000352602 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3355:x1846869322664128:12345-0@lo:400:kworker/u8:0.0 Request processed in 35us (312us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316250.332050:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5332 00000100:00100000:1.0:1761316250.332053:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2627 00000100:00100000:1.0:1761316250.332058:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322664000:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316250.332083:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2628 00000100:00100000:3.0:1761316250.332088:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322664064:12345-0@lo:8:kworker/u8:0.0 00010000:02020000:1.0:1761316250.332335:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316250.332387:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316250.332388:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322664000:12345-0@lo:8:kworker/u8:0.0 Request processed in 329us (650us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316250.332396:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2627 00000100:00100000:3.0:1761316250.332425:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322664064:12345-0@lo:8:kworker/u8:0.0 Request processed in 335us (630us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316250.332432:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2628 00000100:00100000:1.0:1761316250.332439:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002672a249 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322664320:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.332451:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322663744:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.332462:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-OST0001_UUID:3355:1846869322664128:0@lo:400:kworker/u8:0.0 00000100:00080000:3.0:1761316250.332472:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316250.332477:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:1.0:1761316250.332480:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322663936:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316250.332483:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322664000:0@lo:8:kworker/u8:0.0 00000100:00080000:3.0:1761316250.332501:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316250.332504:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316250.332507:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322664064:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316250.332548:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322663808:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.332574:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.332609:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322664192:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316250.332613:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316250.332625:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322664384:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316250.332632:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316250.332653:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761316250.332661:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002c45722b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322664448:0@lo:13:osp-pre-1-1.0 00010000:00010000:2.0:1761316250.332666:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316250.332667:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316250.332688:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663680 00000100:00100000:2.0:1761316250.332717:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1946 00000100:00100000:2.0:1761316250.332721:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322663680:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.336614:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316250.336632:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006a59d025 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322663680:12345-0@lo:400:kworker/u8:0.0 Request processed in 3905us (5053us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316250.336648:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1946 00000100:00100000:0.0:1761316250.336654:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664192 00000100:00100000:1.0:1761316250.336659:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322663872:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.336707:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316250.336710:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5333 00000100:00100000:0.0:1761316250.336717:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000024a9e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322664192:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.336730:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322664256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.336760:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316250.336774:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000024a9e32e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322664192:12345-0@lo:400:kworker/u8:0.0 Request processed in 55us (4144us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316250.336781:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5333 00000100:00100000:2.0:1761316250.336794:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664448 00000100:00100000:0.0:1761316250.336798:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663872 00000100:00100000:1.0:1761316250.336803:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322664192:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316250.336827:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5334 00000100:00100000:2.0:1761316250.336832:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1123 00000100:00100000:0.0:1761316250.336833:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000069f2cc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322663872:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316250.336839:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000596f0dd4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322664448:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316250.336863:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316250.336867:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000069f2cc23 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322663872:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (161us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316250.336874:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5334 00000100:00100000:1.0:1761316250.336886:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322663680:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316250.336903:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000596f0dd4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322664448:12345-0@lo:13:osp-pre-1-1.0 Request processed in 63us (296us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316250.336910:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1123 00000100:00100000:0.0:1761316250.336917:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002c45722b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322664448:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316250.336929:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322663872:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.336941:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664256 00000100:00100000:2.0:1761316250.336946:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322663808 00000100:00100000:1.0:1761316250.336975:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1698 00000100:00100000:2.0:1761316250.336977:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664384 00000100:00100000:1.0:1761316250.336983:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000b653f5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322664256:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316250.337002:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3321 00000100:00100000:2.0:1761316250.337007:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b5a51ecd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322663808:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.337028:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000b653f5d pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322664256:12345-0@lo:400:kworker/u8:0.0 Request processed in 45us (267us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316250.337029:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3322 00000100:00100000:0.0:1761316250.337036:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+7:3355:x1846869322664384:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316250.337040:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1698 00000100:00100000:2.0:1761316250.337043:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b5a51ecd pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322663808:12345-0@lo:400:kworker/u8:0.0 Request processed in 33us (4467us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316250.337050:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3321 00000004:00000024:0.0:1761316250.337077:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:1.0:1761316250.337089:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322664256:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316250.337103:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322663808:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316250.337123:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000017545a58 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322664384:12345-0@lo:41:osp-pre-0-1.0 Request processed in 87us (4460us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316250.337129:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3322 00000100:00100000:2.0:1761316250.341475:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007238283 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322664384:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316251.358435:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316252.359864:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316252.359932:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316252.359975:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316252.360071:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589376 00000100:00100000:3.0:1761316252.360143:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1947 00000100:00100000:3.0:1761316252.360151:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316252.360242:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000091ea913d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589376:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 91us (310us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316252.360251:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1947 00000100:00100000:1.0:1761316252.360391:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589440 00000100:00100000:1.0:1761316252.360436:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589504 00000100:00100000:1.0:1761316252.360461:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5335 00000100:00100000:1.0:1761316252.360467:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307589440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316252.360504:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59cfdd8 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307589440:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 36us (653us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316252.360510:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5335 00000100:00100000:1.0:1761316252.360523:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5336 00000100:00100000:1.0:1761316252.360527:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307589504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316252.360552:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007ba62614 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307589504:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 24us (578us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316252.360558:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5336 00000100:00100000:0.0:1761316252.362739:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316252.363066:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589568 00000100:00100000:2.0:1761316252.363122:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2629 00000100:00100000:2.0:1761316252.363130:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589568:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:2.0:1761316252.363237:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316252.363993:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000049aba795 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307589568:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 861us (1259us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316252.364004:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2629 00010000:00010000:3.0:1761316252.381519:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316252.893128:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322664512:0@lo:41:osp-pre-1-0.0 00000100:00100000:2.0:1761316252.893220:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316252.893384:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664512 00000100:00100000:3.0:1761316252.893455:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3323 00000100:00100000:3.0:1761316252.893462:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000058b5fd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322664512:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316252.893509:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316252.893568:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000058b5fd21 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322664512:12345-0@lo:41:osp-pre-1-0.0 Request processed in 106us (350us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316252.893576:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3323 00000100:00100000:2.0:1761316252.893735:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322664512:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316253.403783:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316254.428617:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316255.452116:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322664640:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316255.452182:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316255.453562:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b54d8406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322664576:0@lo:13:osp-pre-1-1.0 00010000:00010000:3.0:1761316255.453567:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316255.453861:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316255.453900:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322664832:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316255.453930:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316255.454022:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322664768:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316255.454097:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316255.454135:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1761316255.454142:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322664704:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316255.454188:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316255.454239:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664576 00000100:00100000:2.0:1761316255.454304:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1124 00000100:00100000:2.0:1761316255.454312:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000099aed9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322664576:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316255.454376:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316255.454430:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000099aed9bb pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322664576:12345-0@lo:13:osp-pre-1-1.0 Request processed in 117us (570us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316255.454438:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1124 00000100:00100000:2.0:1761316255.454474:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664832 00000100:00100000:2.0:1761316255.454503:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664768 00000100:00100000:2.0:1761316255.454529:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5337 00000100:00100000:2.0:1761316255.454536:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005c22bba0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322664832:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.454559:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664640 00000100:00100000:2.0:1761316255.454577:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005c22bba0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322664832:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (649us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316255.454587:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5337 00000100:00100000:2.0:1761316255.454607:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5338 00000100:00100000:2.0:1761316255.454615:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ab5a1368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322664768:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.454624:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3324 00000100:00100000:0.0:1761316255.454633:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322664640:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316255.454654:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ab5a1368 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322664768:12345-0@lo:400:kworker/u8:2.0 Request processed in 38us (555us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316255.454661:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5338 00000004:00000024:0.0:1761316255.454671:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316255.454713:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000004b40748 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322664768:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.454768:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007238283 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322664640:12345-0@lo:41:osp-pre-0-1.0 Request processed in 133us (2585us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316255.454777:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3324 00000100:00100000:0.0:1761316255.454832:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664704 00000100:00100000:0.0:1761316255.454867:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1948 00000100:00100000:2.0:1761316255.454870:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f6aa85f4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322664832:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.454878:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322664704:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316255.454902:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b54d8406 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322664576:0@lo:13:osp-pre-1-1.0 00000100:00100000:0.0:1761316255.454930:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002133712d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322664704:12345-0@lo:400:kworker/u8:2.0 Request processed in 54us (740us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316255.454938:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1948 00000100:00100000:0.0:1761316255.455001:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001670e17f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322664704:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.455069:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d7057f52 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322664640:0@lo:41:osp-pre-0-1.0 00000100:00080000:3.0:1761316255.455665:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316255.455677:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316255.455682:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316255.455692:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4497 00000100:00080000:3.0:1761316255.455696:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316255.455703:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316255.455731:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009b034734 x1846869322664960/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316255.455751:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316255.455755:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316255.455757:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316255.455761:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4497 00000100:00080000:3.0:1761316255.455764:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316255.455770:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316255.455787:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000b3a868ab x1846869322665024/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761316255.455889:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322664960:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316255.455943:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316255.455967:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322665024:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316255.455992:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316255.456030:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322665152:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316255.456064:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316255.456091:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1761316255.456098:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322665088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316255.456129:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316255.456187:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664960 00000100:00100000:3.0:1761316255.456224:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665024 00000100:00100000:3.0:1761316255.456248:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2630 00000100:00100000:3.0:1761316255.456255:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322664960:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316255.456388:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316255.456439:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322664960:12345-0@lo:8:kworker/u8:2.0 Request processed in 184us (497us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316255.456449:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2630 00000100:00100000:3.0:1761316255.456468:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2631 00000100:00100000:3.0:1761316255.456471:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322665024:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316255.456551:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316255.456582:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000649bcbf9 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322665024:12345-0@lo:8:kworker/u8:2.0 Request processed in 109us (590us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316255.456588:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2631 00000100:00100000:3.0:1761316255.456622:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665152 00000100:00100000:1.0:1761316255.456630:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665088 00000100:00100000:3.0:1761316255.456664:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5339 00000100:00100000:3.0:1761316255.456670:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002672a249 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322665152:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316255.456683:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5340 00000100:00100000:1.0:1761316255.456691:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322665088:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316255.456716:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002672a249 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322665152:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (651us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316255.456725:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5339 00000100:00100000:1.0:1761316255.456769:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001670e17f pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322665088:12345-0@lo:400:kworker/u8:2.0 Request processed in 76us (639us total) trans 0 rc 0/0 00000100:00080000:3.0:1761316255.456780:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1761316255.456781:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5340 00000100:00080000:3.0:1761316255.456789:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316255.456797:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009b034734 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322664960:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761316255.456812:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316255.456815:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316255.456819:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000b3a868ab pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322665024:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316255.456883:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a68dea11 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322665088:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316255.456892:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322665152:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316255.457187:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322665216:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316255.457230:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322664896:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316255.457242:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316255.457293:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316255.462292:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322664896 00000100:00100000:3.0:1761316255.462464:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3325 00000100:00100000:3.0:1761316255.462476:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322664896:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316255.462613:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d7057f52 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322664896:12345-0@lo:400:kworker/u8:2.0 Request processed in 137us (5319us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316255.462625:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3325 00000100:00100000:0.0:1761316255.463016:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322664896:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316255.467825:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665216 00000100:00100000:1.0:1761316255.467910:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1699 00000100:00100000:1.0:1761316255.467917:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322665216:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316255.468024:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322665216:12345-0@lo:400:kworker/u8:2.0 Request processed in 107us (10790us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316255.468032:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1699 00000100:00100000:0.0:1761316255.468257:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322665216:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761316256.476619:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316257.482745:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316257.482795:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316257.482831:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316257.482868:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316257.482975:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589760 00000100:00100000:1.0:1761316257.482995:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589632 00000100:00100000:1.0:1761316257.483048:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1949 00000100:00100000:1.0:1761316257.483054:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307589632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316257.483140:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002d8b084b pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307589632:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 85us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316257.483148:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1949 00000100:00100000:3.0:1761316257.483680:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589824 00000100:00100000:3.0:1761316257.483718:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2632 00000100:00100000:3.0:1761316257.483725:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067d9ad19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307589824:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761316257.483731:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589696 00000100:00100000:0.0:1761316257.483782:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5341 00000100:00100000:0.0:1761316257.483789:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307589760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00010000:02020000:3.0:1761316257.483818:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316257.483878:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008859f1fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307589760:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 81us (1040us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316257.483880:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067d9ad19 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307589824:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 147us (1134us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316257.483901:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5341 00000100:00100000:3.0:1761316257.483904:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2632 00000100:00100000:3.0:1761316257.483945:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5342 00000100:00100000:3.0:1761316257.483951:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307589696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316257.483986:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d332521 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307589696:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 35us (1119us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316257.483992:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5342 00010000:00010000:0.0:1761316257.499608:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316258.012898:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322665280:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316258.012972:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316258.014668:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665280 00000100:00100000:1.0:1761316258.015061:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3326 00000100:00100000:1.0:1761316258.015067:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ba8be421 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322665280:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316258.015102:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316258.015172:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ba8be421 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322665280:12345-0@lo:41:osp-pre-1-0.0 Request processed in 104us (2201us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316258.015179:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3326 00000100:00100000:0.0:1761316258.015627:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322665280:0@lo:41:osp-pre-1-0.0 00010000:00010000:3.0:1761316258.525824:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316259.554656:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316260.571573:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316260.571755:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322665344:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316260.571820:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316260.571862:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1761316260.571867:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dfd01862 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322665408:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316260.571890:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316260.572061:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665344 00000100:00100000:0.0:1761316260.572106:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3327 00000100:00100000:0.0:1761316260.572112:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322665344:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316260.572144:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316260.572182:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f8b0b3b0 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3353:x1846869322665344:12345-0@lo:41:osp-pre-0-1.0 Request processed in 68us (361us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316260.572188:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3327 00000100:00100000:1.0:1761316260.572786:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322665344:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316260.573093:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665408 00000100:00100000:2.0:1761316260.573135:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1125 00000100:00100000:2.0:1761316260.573139:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322665408:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316260.573160:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316260.573210:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e5496139 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3353:x1846869322665408:12345-0@lo:13:osp-pre-1-1.0 Request processed in 70us (1319us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316260.573216:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1125 00000100:00100000:2.0:1761316260.573522:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322665472:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316260.573555:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316260.573632:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dfd01862 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322665408:0@lo:13:osp-pre-1-1.0 00000100:00100000:3.0:1761316260.574066:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665472 00000100:00100000:3.0:1761316260.574100:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1950 00000100:00100000:3.0:1761316260.574105:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322665472:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.574411:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005b1075d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322665536:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.574443:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316260.574480:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665536 00000100:00100000:1.0:1761316260.574509:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5343 00000100:00100000:1.0:1761316260.574513:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322665536:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.574547:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001a75c6c2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3356:x1846869322665536:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (104us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316260.574553:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5343 00000100:00100000:1.0:1761316260.574589:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005b1075d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-MDT0000_UUID:3356:1846869322665536:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761316260.574750:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316260.574755:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316260.574758:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316260.574763:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4503 00000100:00080000:0.0:1761316260.574766:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316260.574769:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316260.574787:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009404a1e5 x1846869322665728/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761316260.574801:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1761316260.574804:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316260.574805:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316260.574808:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4503 00000100:00080000:0.0:1761316260.574809:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316260.574812:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316260.574821:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005c899dd9 x1846869322665792/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316260.574861:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322665728:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316260.574897:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316260.574913:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322665792:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316260.574934:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316260.574974:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665728 00000100:00100000:0.0:1761316260.574996:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665792 00000100:00100000:0.0:1761316260.575011:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2633 00000100:00100000:0.0:1761316260.575015:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d9a801e7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322665728:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575392:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000001027968 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322665472:12345-0@lo:400:kworker/u8:2.0 Request processed in 1285us (1836us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316260.575400:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1950 00000100:00100000:3.0:1761316260.575436:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2634 00000100:00100000:3.0:1761316260.575440:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322665792:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316260.575520:0:13311:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316260.575560:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000067cdb3e5 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:0+-99:3352:x1846869322665792:12345-0@lo:8:kworker/u8:2.0 Request processed in 119us (626us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316260.575566:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2634 00000100:00100000:3.0:1761316260.575599:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322665664:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575624:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316260.575643:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1761316260.575649:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322665600:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575670:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316260.575683:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000793073e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322665984:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575702:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316260.575721:0:13311:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665984 00000100:00100000:3.0:1761316260.575746:0:13311:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1700 00000100:00100000:3.0:1761316260.575750:0:13311:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322665984:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575781:0:13311:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009b034734 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_004:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322665984:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (79us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316260.575787:0:13311:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1700 00000100:00100000:3.0:1761316260.575814:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000793073e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322665984:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575834:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665664 00000100:00100000:3.0:1761316260.575856:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3328 00000100:00100000:3.0:1761316260.575860:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322665664:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575889:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a68dea11 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322665664:12345-0@lo:400:kworker/u8:2.0 Request processed in 29us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316260.575895:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3328 00000100:00100000:3.0:1761316260.575915:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322665664:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575933:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cdfd94f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322665920:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.575973:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316260.575989:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:3.0:1761316260.575994:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322665856:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316260.576013:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316260.577759:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665600 00000100:00100000:1.0:1761316260.577811:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665920 00000100:00100000:1.0:1761316260.577826:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322665856 00000100:00100000:1.0:1761316260.577847:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5344 00000100:00100000:1.0:1761316260.577852:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b3a868ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322665600:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.577924:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b3a868ab pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322665600:12345-0@lo:400:kworker/u8:2.0 Request processed in 71us (2253us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316260.577930:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5344 00000100:00100000:1.0:1761316260.577957:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5345 00000100:00100000:1.0:1761316260.577960:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f6aa85f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322665920:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.577993:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f6aa85f4 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322665920:12345-0@lo:400:kworker/u8:2.0 Request processed in 31us (2020us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316260.577998:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5345 00000100:00100000:1.0:1761316260.578007:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5346 00000100:00100000:1.0:1761316260.578010:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000004b40748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322665856:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.578034:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000004b40748 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322665856:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (2021us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316260.578040:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5346 00000100:00100000:1.0:1761316260.578097:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322665600:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.578140:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322665856:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316260.578148:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cdfd94f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322665920:0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761316260.579402:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316260.579460:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d9a801e7 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322665728:12345-0@lo:8:kworker/u8:2.0 Request processed in 4444us (4562us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316260.579470:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2633 00000100:00080000:3.0:1761316260.580055:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316260.580061:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316260.580065:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322665728:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761316260.580077:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316260.580079:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316260.580081:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322665792:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316260.580496:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d131fbc7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322665472:0@lo:400:kworker/u8:2.0 00010000:00010000:1.0:1761316261.598870:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316262.600564:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316262.600643:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589888 00000100:00100000:0.0:1761316262.600705:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1951 00000100:00100000:0.0:1761316262.600712:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316262.600782:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000042df9a96 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307589888:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 70us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316262.600789:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1951 00000100:00100000:3.0:1761316262.606757:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316262.607723:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316262.607779:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316262.607871:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307589952 00000100:00100000:1.0:1761316262.607920:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590016 00000100:00100000:1.0:1761316262.607939:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5347 00000100:00100000:1.0:1761316262.607945:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307589952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316262.608011:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000003430de1 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2266:x1846869307589952:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 65us (1261us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316262.608019:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5347 00000100:00100000:1.0:1761316262.608034:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5348 00000100:00100000:1.0:1761316262.608037:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307590016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316262.608055:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000034aae76b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307590016:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 18us (277us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316262.608060:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5348 00000100:00100000:0.0:1761316262.608315:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590080 00000100:00100000:0.0:1761316262.608361:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2635 00000100:00100000:0.0:1761316262.608366:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590080:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:0.0:1761316262.608455:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316262.608493:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000097937f1c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590080:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 126us (771us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316262.608499:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2635 00010000:00010000:2.0:1761316262.619608:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316263.132068:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322666048:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316263.132131:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316263.132506:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666048 00000100:00100000:3.0:1761316263.132721:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3329 00000100:00100000:3.0:1761316263.132726:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322666048:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316263.132756:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316263.132802:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c03a86 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322666048:12345-0@lo:41:osp-pre-1-0.0 Request processed in 75us (671us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316263.132809:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3329 00000100:00100000:1.0:1761316263.135496:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322666048:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316263.643660:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1761316264.667676:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316265.692329:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322666112:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316265.692419:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316265.692513:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666112 00000100:00100000:2.0:1761316265.692561:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1126 00000100:00100000:2.0:1761316265.692567:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322666112:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316265.692603:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316265.692640:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000522c807b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322666112:12345-0@lo:13:osp-pre-1-1.0 Request processed in 72us (223us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316265.692647:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1126 00000100:00100000:1.0:1761316265.693010:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001c97fc10 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322666112:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316265.693401:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316265.693406:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316265.693410:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316265.693414:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4508 00000100:00080000:3.0:1761316265.693416:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316265.693420:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316265.693439:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e334a674 x1846869322666496/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316265.693454:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316265.693457:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316265.693458:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316265.693461:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4508 00000100:00080000:3.0:1761316265.693463:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1761316265.693464:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322666304:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316265.693465:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:3.0:1761316265.693477:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007e8c11e2 x1846869322666560/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316265.693495:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316265.693517:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a966e8fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322666688:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.693522:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322666496:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693539:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316265.693546:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316265.693552:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1761316265.693556:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322666240:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.693559:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007e8c11e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322666560:0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693575:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316265.693575:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316265.693586:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000befc837e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322666624:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693604:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316265.693614:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666496 00000100:00100000:1.0:1761316265.693625:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666688 00000100:00100000:3.0:1761316265.693633:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666560 00000100:00100000:1.0:1761316265.693646:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666240 00000100:00100000:3.0:1761316265.693646:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2636 00000100:00100000:3.0:1761316265.693650:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322666496:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693658:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666624 00000100:00100000:1.0:1761316265.693670:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5349 00000100:00100000:1.0:1761316265.693673:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322666688:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693709:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000727dca98 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0001-lwp-OST0001_UUID+5:3354:x1846869322666688:12345-0@lo:400:kworker/u8:2.0 Request processed in 35us (170us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316265.693715:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5349 00000100:00100000:1.0:1761316265.693727:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5350 00000100:00100000:1.0:1761316265.693730:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322666240:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761316265.693738:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316265.693754:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc9be46d pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-MDT0000_UUID+5:3354:x1846869322666240:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (180us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316265.693760:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5350 00000100:00100000:3.0:1761316265.693763:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000da32678b pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322666496:12345-0@lo:8:kworker/u8:2.0 Request processed in 113us (218us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316265.693768:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2636 00000100:00100000:1.0:1761316265.693770:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5351 00000100:00100000:1.0:1761316265.693773:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322666624:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.693778:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2637 00000100:00100000:3.0:1761316265.693780:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322666560:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693793:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa1b59d6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322666624:12345-0@lo:400:kworker/u8:2.0 Request processed in 20us (190us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316265.693798:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5351 00000100:00100000:1.0:1761316265.693836:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000649bcbf9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0000_UUID:3354:1846869322666240:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693843:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000befc837e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322666624:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693850:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a966e8fa pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-lwp-OST0001_UUID:3354:1846869322666688:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693881:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322666368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693900:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316265.693911:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f5c0c3f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322666752:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.693926:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316265.693939:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:1.0:1761316265.693944:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322666176:0@lo:400:kworker/u8:2.0 00010000:02020000:3.0:1761316265.693948:0:8227:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316265.693964:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316265.693970:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ebd8a2b4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:0+-99:3352:x1846869322666560:12345-0@lo:8:kworker/u8:2.0 Request processed in 188us (396us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316265.693975:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322666432:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.693975:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2637 00000100:00100000:3.0:1761316265.693986:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666752 00000100:00100000:1.0:1761316265.693991:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316265.694005:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1701 00000100:00100000:3.0:1761316265.694008:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322666752:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.694019:0:9703:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666368 00000100:00100000:3.0:1761316265.694036:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b59a8b32 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322666752:12345-0@lo:400:kworker/u8:2.0 Request processed in 27us (109us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316265.694037:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5352 00000100:00100000:1.0:1761316265.694040:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322666368:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.694045:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1701 00000100:00100000:3.0:1761316265.694063:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666304 00000100:00100000:1.0:1761316265.694064:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007d77d078 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322666368:12345-0@lo:400:kworker/u8:2.0 Request processed in 23us (164us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316265.694069:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5352 00000100:00100000:1.0:1761316265.694092:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004c227bf8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322666368:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.694099:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f5c0c3f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322666752:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.695297:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666432 00000100:00100000:3.0:1761316265.695315:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3330 00000100:00100000:3.0:1761316265.695320:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322666304:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.695382:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d733c02e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322666304:12345-0@lo:400:kworker/u8:2.0 Request processed in 61us (1886us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316265.695388:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3330 00000100:00100000:3.0:1761316265.695400:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3331 00000100:00100000:3.0:1761316265.695403:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322666432:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316265.695427:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000bba19120 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322666304:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.695434:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f694cb8 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322666432:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (1443us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316265.695439:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3331 00000100:00080000:3.0:1761316265.695463:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1761316265.695464:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000001014ca3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322666432:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316265.695467:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316265.695469:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e334a674 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322666496:0@lo:8:kworker/u8:2.0 00000100:00080000:3.0:1761316265.695477:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316265.695480:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316265.695482:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007e8c11e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322666560:0@lo:8:kworker/u8:2.0 00000100:00100000:2.0:1761316265.695895:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:2.0:1761316265.696728:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322666816:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316265.696781:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316265.696898:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666176 00000100:00100000:0.0:1761316265.696936:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1952 00000100:00100000:0.0:1761316265.696940:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322666176:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316265.696983:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000abbf023a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322666176:12345-0@lo:400:kworker/u8:2.0 Request processed in 42us (3019us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316265.696990:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1952 00010000:00010000:0.0:1761316265.697055:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316265.698112:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000af1aef25 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322666176:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316265.698597:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666816 00000100:00100000:3.0:1761316265.698647:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3332 00000100:00100000:3.0:1761316265.698652:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322666816:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:3.0:1761316265.698683:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:3.0:1761316265.698743:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000cf0de5dc pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322666816:12345-0@lo:41:osp-pre-0-1.0 Request processed in 90us (1963us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316265.698750:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3332 00000100:00100000:1.0:1761316265.698825:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322666816:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316266.715502:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1761316267.721001:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316267.721041:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316267.721077:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316267.721105:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590144 00000100:00100000:3.0:1761316267.721116:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316267.721117:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590208 00000100:00100000:3.0:1761316267.721146:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590336 00000100:00100000:0.0:1761316267.721168:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1953 00000100:00100000:2.0:1761316267.721177:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590272 00000100:00100000:3.0:1761316267.721181:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2638 00000100:00100000:0.0:1761316267.721186:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307590144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316267.721190:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590336:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00000100:00100000:2.0:1761316267.721231:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5353 00000100:00100000:2.0:1761316267.721237:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007e8c11e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307590208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:2.0:1761316267.721286:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007e8c11e2 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307590208:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 43us (240us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316267.721288:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000693184de pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2266:x1846869307590144:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 104us (287us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316267.721300:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5353 00010000:02020000:3.0:1761316267.721306:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316267.721315:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1953 00000100:00100000:2.0:1761316267.721323:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5354 00000100:00100000:2.0:1761316267.721330:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307590272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:3.0:1761316267.721338:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000801bff56 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590336:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 148us (261us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316267.721376:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2638 00000100:00100000:2.0:1761316267.721398:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000f1ae97ad pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2266:x1846869307590272:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 68us (283us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316267.721407:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5354 00010000:00010000:1.0:1761316267.739593:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316268.251785:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322666880:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316268.251858:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316268.251977:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666880 00000100:00100000:1.0:1761316268.252031:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3333 00000100:00100000:1.0:1761316268.252037:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322666880:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316268.252070:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316268.252125:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002513e550 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322666880:12345-0@lo:41:osp-pre-1-0.0 Request processed in 88us (267us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316268.252132:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3333 00000100:00100000:0.0:1761316268.252233:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000007c7b302 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322666880:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316268.763534:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316269.787521:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316270.811684:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322666944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316270.811762:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316270.811876:0:8231:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322666944 00000100:00100000:2.0:1761316270.811931:0:8231:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1127 00000100:00100000:2.0:1761316270.811937:0:8231:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322666944:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:2.0:1761316270.811965:0:8231:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316270.812004:0:8231:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a790d12c pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322666944:12345-0@lo:13:osp-pre-1-1.0 Request processed in 67us (244us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316270.812011:0:8231:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1127 00000100:00100000:1.0:1761316270.812095:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c64d6e0e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322666944:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316270.812828:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c6808c98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322667008:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316270.812860:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:1.0:1761316270.812900:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761316270.812989:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316270.812993:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316270.812996:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316270.813001:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4513 00000100:00080000:2.0:1761316270.813003:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761316270.813007:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316270.813023:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000afc8cc98 x1846869322667328/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761316270.813035:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1761316270.813038:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1761316270.813039:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1761316270.813042:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4513 00000100:00080000:2.0:1761316270.813043:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1761316270.813045:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316270.813053:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000028e6bf3e x1846869322667392/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316270.813118:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322667072:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.813147:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316270.813171:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322667456:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.813203:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316270.813222:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:1.0:1761316270.813227:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052a02902 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322667264:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.813247:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316270.813274:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667456 00000100:00100000:1.0:1761316270.813302:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5355 00000100:00100000:1.0:1761316270.813307:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322667456:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.813536:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322667328:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316270.813763:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667008 00000100:00100000:0.0:1761316270.813799:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667264 00000100:00100000:0.0:1761316270.813817:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3334 00000100:00100000:0.0:1761316270.813821:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322667008:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316270.813856:0:6259:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316270.813957:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667072 00000100:00100000:2.0:1761316270.813988:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1954 00000100:00100000:2.0:1761316270.813992:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322667072:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.814211:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000057deb555 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322667456:12345-0@lo:400:kworker/u8:2.0 Request processed in 903us (1007us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316270.814219:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5355 00000100:00100000:1.0:1761316270.814245:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3335 00000100:00100000:1.0:1761316270.814249:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322667264:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.814282:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000077de439a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3356:x1846869322667264:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (1035us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316270.814287:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3335 00000100:00100000:1.0:1761316270.814375:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052a02902 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-mdtlov_UUID:3356:1846869322667264:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.814456:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316270.814478:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322667392:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316270.814507:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7274d4a pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0001-mdtlov_UUID+6:3355:x1846869322667008:12345-0@lo:41:osp-pre-0-1.0 Request processed in 682us (1644us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316270.814508:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316270.814518:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3334 00000100:00100000:0.0:1761316270.814546:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667328 00000100:00100000:3.0:1761316270.814560:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667392 00000100:00100000:0.0:1761316270.814570:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2639 00000100:00100000:0.0:1761316270.814574:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000616e67fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322667328:12345-0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316270.814598:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2640 00000100:00100000:3.0:1761316270.814603:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322667392:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316270.814689:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316270.814728:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000038c20d21 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:3352:x1846869322667392:12345-0@lo:8:kworker/u8:2.0 Request processed in 124us (223us total) trans 0 rc -19/-19 00000100:00100000:1.0:1761316270.814733:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ce0031ff pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322667456:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.814737:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2640 00000100:00100000:3.0:1761316270.814768:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322667136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.814778:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322667200:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.814797:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316270.814818:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322667520:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.814819:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316270.814843:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ed1079bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322667584:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.814866:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316270.814871:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316270.814905:0:8227:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667584 00000100:00100000:1.0:1761316270.814915:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667136 00000100:00100000:3.0:1761316270.814934:0:8227:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1702 00000100:00100000:3.0:1761316270.814939:0:8227:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322667584:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.814947:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667200 00000100:00100000:1.0:1761316270.814970:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667520 00000100:00100000:3.0:1761316270.814982:0:8227:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bd4b7950 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_000:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322667584:12345-0@lo:400:kworker/u8:2.0 Request processed in 41us (113us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316270.814990:0:8227:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1702 00000100:00100000:1.0:1761316270.814995:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5356 00000100:00100000:1.0:1761316270.815001:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322667136:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316270.815014:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5357 00000100:00100000:2.0:1761316270.815024:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b207e359 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3356:x1846869322667072:12345-0@lo:400:kworker/u8:2.0 Request processed in 1027us (1872us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316270.815024:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322667200:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316270.815040:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1954 00000100:00100000:1.0:1761316270.815047:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000dc5d609 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322667136:12345-0@lo:400:kworker/u8:2.0 Request processed in 44us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316270.815055:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5356 00000100:00100000:3.0:1761316270.815072:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004f9b8f16 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322667200:12345-0@lo:400:kworker/u8:2.0 Request processed in 50us (254us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316270.815077:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5358 00000100:00100000:1.0:1761316270.815082:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322667520:12345-0@lo:400:kworker/u8:2.0 00010000:02020000:0.0:1761316270.815156:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316270.815192:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000616e67fa pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322667328:12345-0@lo:8:kworker/u8:2.0 Request processed in 617us (737us total) trans 0 rc -19/-19 00000100:00100000:0.0:1761316270.815198:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2639 00000100:00100000:0.0:1761316270.815234:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c6808c98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322667008:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316270.815277:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000603a95fb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322667200:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316270.815284:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ed1079bc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322667584:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316270.815326:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000a254e2e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:557e7bf9-3fe3-4784-8f78-b1773a90f048:3356:1846869322667072:0@lo:400:kworker/u8:2.0 00000100:00080000:0.0:1761316270.815443:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316270.815447:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316270.815452:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000afc8cc98 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322667328:0@lo:8:kworker/u8:2.0 00000100:00080000:0.0:1761316270.815462:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316270.815464:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316270.815467:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028e6bf3e pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322667392:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316270.815472:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5357 00000100:00100000:3.0:1761316270.815517:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000275a380d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322667136:0@lo:400:kworker/u8:2.0 00000100:00100000:1.0:1761316270.815568:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fa2b8875 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322667520:12345-0@lo:400:kworker/u8:2.0 Request processed in 484us (717us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316270.815576:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5358 00000100:00100000:3.0:1761316270.815740:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000290640a4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322667520:0@lo:400:kworker/u8:2.0 00010000:00010000:0.0:1761316271.835543:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316272.842206:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316272.844410:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590464 00000100:00100000:2.0:1761316272.844483:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316272.844533:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316272.844573:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590528 00000100:00100000:2.0:1761316272.844603:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5359 00000100:00100000:2.0:1761316272.844612:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307590464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316272.844722:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b86079b3 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307590464:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 111us (2529us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316272.844731:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5359 00000100:00100000:2.0:1761316272.844750:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5360 00000100:00100000:2.0:1761316272.844755:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307590528:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:2.0:1761316272.844780:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce6ae3b6 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307590528:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 25us (248us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316272.844787:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5360 00000100:00100000:2.0:1761316272.844980:0:3347:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:0.0:1761316272.845243:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590400 00000100:00100000:0.0:1761316272.845440:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1955 00000100:00100000:0.0:1761316272.845446:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307590400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316272.845498:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bc496c9d pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307590400:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 50us (1018us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316272.845505:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1955 00000100:00100000:3.0:1761316272.845613:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590592 00000100:00100000:3.0:1761316272.845640:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2641 00000100:00100000:3.0:1761316272.845645:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590592:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00010000:02020000:3.0:1761316272.845743:0:17018:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316272.845775:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006766d617 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:0+-99:2262:x1846869307590592:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 130us (796us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316272.845781:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2641 00010000:00010000:0.0:1761316272.859544:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316273.371789:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093968913 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322667648:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316273.371895:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316273.372087:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667648 00000100:00100000:2.0:1761316273.372163:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3336 00000100:00100000:2.0:1761316273.372179:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322667648:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316273.372227:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316273.372290:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c6808c98 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322667648:12345-0@lo:41:osp-pre-1-0.0 Request processed in 118us (396us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316273.372297:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3336 00000100:00100000:0.0:1761316273.372562:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093968913 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322667648:0@lo:41:osp-pre-1-0.0 00010000:00010000:2.0:1761316273.883608:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316274.907585:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316275.931526:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000095732ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322667712:0@lo:41:osp-pre-0-1.0 00010000:00010000:0.0:1761316275.931568:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316275.931597:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316275.931730:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667712 00000100:00100000:0.0:1761316275.931793:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3337 00000100:00100000:0.0:1761316275.931801:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322667712:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:0.0:1761316275.931835:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00080000:3.0:1761316275.931851:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316275.931853:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322667840:0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316275.931857:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316275.931862:0:37:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316275.931868:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4518 00000100:00080000:3.0:1761316275.931871:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316275.931876:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316275.931881:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008e76633c pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322667712:12345-0@lo:41:osp-pre-0-1.0 Request processed in 80us (285us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316275.931889:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3337 00000100:00100000:2.0:1761316275.931895:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316275.931898:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000033e21383 x1846869322668096/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316275.931915:0:37:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1761316275.931917:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322667904:0@lo:400:kworker/u8:2.0 00000100:00080000:3.0:1761316275.931920:0:37:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316275.931922:0:37:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761316275.931927:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00080000:3.0:1761316275.931927:0:37:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4518 00000100:00080000:3.0:1761316275.931931:0:37:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316275.931934:0:37:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316275.931937:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000421e79db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322667776:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.931948:0:37:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009d6dfec7 x1846869322668160/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316275.931957:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316275.931975:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316275.931989:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322668288:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316275.932000:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c56478c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322668032:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.932011:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322668096:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316275.932026:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316275.932033:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316275.932051:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [3->2] 00000100:00100000:3.0:1761316275.932056:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316275.932060:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322667968:0@lo:400:kworker/u8:2.0 00000100:00100000:2.0:1761316275.932062:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322668224:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.932078:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009d6dfec7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322668160:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316275.932099:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316275.932112:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668096 00000100:00100000:2.0:1761316275.932169:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2642 00000100:00100000:2.0:1761316275.932173:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322668096:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:2.0:1761316275.932259:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316275.932906:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668032 00000100:00100000:1.0:1761316275.932941:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3338 00000100:00100000:1.0:1761316275.932945:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322668032:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.933476:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316275.933527:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668160 00000100:00100000:3.0:1761316275.933551:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2643 00000100:00100000:3.0:1761316275.933555:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322668160:12345-0@lo:8:kworker/u8:2.0 00010000:02020000:3.0:1761316275.933641:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:3.0:1761316275.933676:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000008ad8b822 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322668160:12345-0@lo:8:kworker/u8:2.0 Request processed in 120us (201us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316275.933684:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2643 00000100:00080000:3.0:1761316275.933721:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316275.933725:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316275.933729:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009d6dfec7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322668160:0@lo:8:kworker/u8:2.0 00000100:00100000:3.0:1761316275.933764:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316275.933801:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667904 00000100:00100000:3.0:1761316275.933825:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668288 00000100:00100000:3.0:1761316275.933841:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667968 00000100:00100000:3.0:1761316275.933850:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668224 00000100:00100000:3.0:1761316275.933862:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5361 00000100:00100000:3.0:1761316275.933866:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322667904:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.933900:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf5b9d47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322667904:12345-0@lo:400:kworker/u8:2.0 Request processed in 33us (1949us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316275.933907:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5361 00000100:00100000:3.0:1761316275.933918:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5362 00000100:00100000:3.0:1761316275.933921:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322668288:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.933947:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000062377729 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0001-lwp-OST0001_UUID+5:3355:x1846869322668288:12345-0@lo:400:kworker/u8:2.0 Request processed in 25us (1923us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316275.933953:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5362 00000100:00100000:3.0:1761316275.933962:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5363 00000100:00100000:3.0:1761316275.933965:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322667968:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.933988:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ee31279c pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0001_UUID+5:3355:x1846869322667968:12345-0@lo:400:kworker/u8:2.0 Request processed in 22us (1891us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316275.933992:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5363 00000100:00100000:3.0:1761316275.934001:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5364 00000100:00100000:3.0:1761316275.934005:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322668224:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.934030:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009d6dfec7 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3354:x1846869322668224:12345-0@lo:400:kworker/u8:2.0 Request processed in 24us (265us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316275.934034:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5364 00000100:00100000:3.0:1761316275.934064:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-OST0001_UUID:3354:1846869322668224:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.934119:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667776 00000100:00100000:0.0:1761316275.934433:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0001_UUID:3355:1846869322667968:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934443:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322668352:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934472:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316275.934495:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-lwp-OST0001_UUID:3355:1846869322668288:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934506:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322667904:0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934530:0:17018:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668352 00000100:00100000:0.0:1761316275.934555:0:17018:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1703 00000100:00100000:0.0:1761316275.934559:0:17018:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322668352:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934590:0:17018:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000003226e4c2 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_005:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322668352:12345-0@lo:400:kworker/u8:2.0 Request processed in 30us (118us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316275.934596:0:17018:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1703 00000100:00100000:2.0:1761316275.934609:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000009df8a936 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322668096:12345-0@lo:8:kworker/u8:2.0 Request processed in 2434us (2555us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316275.934616:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2642 00000100:00100000:0.0:1761316275.934618:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322668352:0@lo:400:kworker/u8:2.0 00000100:00080000:2.0:1761316275.934642:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761316275.934645:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:2.0:1761316275.934648:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322668096:0@lo:8:kworker/u8:2.0 00000100:00100000:0.0:1761316275.934668:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000095732ed pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322667712:0@lo:41:osp-pre-0-1.0 00000100:00100000:0.0:1761316275.934692:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322667840 00000100:00100000:0.0:1761316275.934714:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1128 00000100:00100000:0.0:1761316275.934717:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322667840:12345-0@lo:13:osp-pre-1-1.0 00002000:00000024:0.0:1761316275.934736:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:0.0:1761316275.934776:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001989a362 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3354:x1846869322667840:12345-0@lo:13:osp-pre-1-1.0 Request processed in 58us (2883us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316275.934782:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1128 00000100:00100000:2.0:1761316275.934898:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000748e0545 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322667840:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316275.936642:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007b71f48d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322668032:12345-0@lo:400:kworker/u8:2.0 Request processed in 3689us (4604us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316275.936655:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3338 00000100:00100000:2.0:1761316275.936860:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c56478c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322668032:0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.937961:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1956 00000100:00100000:3.0:1761316275.937970:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322667776:12345-0@lo:400:kworker/u8:2.0 00000100:00100000:3.0:1761316275.938055:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001f80e34a pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3354:x1846869322667776:12345-0@lo:400:kworker/u8:2.0 Request processed in 86us (6083us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316275.938062:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1956 00000100:00100000:0.0:1761316275.938327:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000421e79db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:557e7bf9-3fe3-4784-8f78-b1773a90f048:3354:1846869322667776:0@lo:400:kworker/u8:2.0 00010000:00010000:2.0:1761316276.956705:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316277.961509:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316277.962030:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590848 00000100:00100000:3.0:1761316277.962608:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2644 00000100:00100000:3.0:1761316277.962616:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307590848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 00010000:02020000:3.0:1761316277.962713:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316277.966630:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316277.966668:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316277.966695:0:3346:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:1.0:1761316277.966733:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590784 00000100:00100000:1.0:1761316277.966777:0:6254:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590720 00000100:00100000:1.0:1761316277.966797:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5365 00000100:00100000:1.0:1761316277.966803:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307590784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316277.966861:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000fe1c4fae pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2265:x1846869307590784:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 57us (193us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316277.966868:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5365 00000100:00100000:1.0:1761316277.966884:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5366 00000100:00100000:1.0:1761316277.966887:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307590720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:1.0:1761316277.966908:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000093a69a62 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2265:x1846869307590720:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 20us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316277.966913:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5366 00000100:00100000:3.0:1761316277.967008:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000748e0545 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:2262:x1846869307590848:12345-192.168.203.51@tcp:8:kworker/u8:1.0 Request processed in 4391us (5506us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316277.967017:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2644 00000100:00100000:0.0:1761316277.967400:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590656 00000100:00100000:0.0:1761316277.967434:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1957 00000100:00100000:0.0:1761316277.967439:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307590656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 00000100:00100000:0.0:1761316277.967465:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002eff52a6 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2265:x1846869307590656:12345-192.168.203.51@tcp:400:kworker/u8:1.0 Request processed in 25us (843us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316277.967471:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1957 00010000:00010000:0.0:1761316277.980115:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316278.492062:0:3354:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1761316278.492087:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322668416:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316278.492184:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316278.492245:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668416 00000100:00100000:3.0:1761316278.492296:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3339 00000100:00100000:3.0:1761316278.492302:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322668416:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:3.0:1761316278.492334:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:3.0:1761316278.492401:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000095732ed pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322668416:12345-0@lo:41:osp-pre-1-0.0 Request processed in 98us (218us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316278.492411:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3339 00000100:00100000:0.0:1761316278.492513:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322668416:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316279.003526:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1761316280.027856:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316281.051513:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322668480:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316281.051543:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322668544:0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316281.051583:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00010000:00010000:0.0:1761316281.051586:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316281.051589:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316281.051649:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668480 00000100:00100000:2.0:1761316281.051694:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3340 00000100:00100000:2.0:1761316281.051700:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322668480:12345-0@lo:41:osp-pre-0-1.0 00000004:00000024:2.0:1761316281.051735:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:2.0:1761316281.051775:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e7bedaeb pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0001-mdtlov_UUID+6:3354:x1846869322668480:12345-0@lo:41:osp-pre-0-1.0 Request processed in 74us (193us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316281.051783:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3340 00000100:00100000:2.0:1761316281.051870:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668544 00000100:00100000:2.0:1761316281.051901:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1129 00000100:00100000:2.0:1761316281.051906:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322668544:12345-0@lo:13:osp-pre-1-1.0 00000100:00080000:3.0:1761316281.051922:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00002000:00000024:2.0:1761316281.051925:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00080000:3.0:1761316281.051928:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316281.051932:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316281.051936:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4523 00000100:00080000:3.0:1761316281.051940:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316281.051944:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:2.0:1761316281.052057:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000290640a4 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3355:x1846869322668544:12345-0@lo:13:osp-pre-1-1.0 Request processed in 149us (469us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316281.052065:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1129 00000100:00100000:2.0:1761316281.052089:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322668800:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052124:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052163:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668800 00000100:00100000:2.0:1761316281.052174:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d6897f1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0001-mdtlov_UUID:3355:1846869322668544:0@lo:13:osp-pre-1-1.0 00000100:00100000:2.0:1761316281.052192:0:3355:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [3->2] 00000100:00100000:0.0:1761316281.052194:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3341 00000100:00100000:2.0:1761316281.052199:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322668608:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052202:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322668800:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052237:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052250:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000275a380d pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322668800:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (124us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316281.052258:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3341 00000100:00100000:2.0:1761316281.052265:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000033e21383 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322668800:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316281.052291:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007c56478c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322668736:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052295:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668608 00000100:00100000:2.0:1761316281.052314:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000421e79db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322668672:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052326:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1958 00000100:00100000:0.0:1761316281.052331:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322668608:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052373:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052413:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000a3e70612 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3355:x1846869322668608:12345-0@lo:400:kworker/u8:0.0 Request processed in 81us (177us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316281.052421:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1958 00000100:00100000:3.0:1761316281.052436:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db207c22 x1846869322668864/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761316281.052456:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1761316281.052460:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1761316281.052462:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1761316281.052466:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4523 00000100:00080000:3.0:1761316281.052468:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1761316281.052472:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316281.052480:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c1516f67 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:557e7bf9-3fe3-4784-8f78-b1773a90f048:3355:1846869322668608:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316281.052488:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009fcb6e1b x1846869322668928/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:0.0:1761316281.052520:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668672 00000100:00100000:2.0:1761316281.052529:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322668864:0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052553:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5367 00000100:00100000:3.0:1761316281.052557:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322668992:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052561:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322668672:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052567:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316281.052590:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322668928:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316281.052603:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052611:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000d6897f1b pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0000_UUID+5:3353:x1846869322668672:12345-0@lo:400:kworker/u8:0.0 Request processed in 51us (238us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316281.052620:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5367 00000100:00100000:2.0:1761316281.052625:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052638:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668992 00000100:00100000:3.0:1761316281.052654:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668864 00000100:00100000:0.0:1761316281.052668:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5368 00000100:00100000:0.0:1761316281.052674:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322668992:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052674:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322669056:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316281.052678:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668928 00000100:00100000:1.0:1761316281.052683:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316281.052696:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2645 00000100:00100000:3.0:1761316281.052700:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322668864:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316281.052715:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0001-mdtlov_UUID:3354:1846869322668480:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316281.052722:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052729:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000007c7b302 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322668992:12345-0@lo:400:kworker/u8:0.0 Request processed in 54us (127us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316281.052737:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000093968913 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322669120:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052739:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5368 00000100:00100000:2.0:1761316281.052755:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000421e79db pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0000_UUID:3353:1846869322668672:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052759:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322668736 00000100:00100000:1.0:1761316281.052768:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316281.052780:0:13140:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669056 00010000:02020000:3.0:1761316281.052784:0:8228:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316281.052800:0:13140:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5369 00000100:00100000:0.0:1761316281.052806:0:13140:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322668736:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316281.052813:0:9703:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5370 00000100:00100000:2.0:1761316281.052813:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669120 00000100:00100000:1.0:1761316281.052821:0:9703:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000a254e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322669056:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316281.052821:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@0000000028e6bf3e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:0+-99:3352:x1846869322668864:12345-0@lo:8:kworker/u8:0.0 Request processed in 120us (258us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316281.052828:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2645 00000100:00100000:3.0:1761316281.052844:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1704 00000100:00100000:3.0:1761316281.052849:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322669120:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052854:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2646 00000100:00100000:0.0:1761316281.052857:0:13140:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000006026bd47 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_004:lustre-MDT0000-lwp-MDT0001_UUID+5:3354:x1846869322668736:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (176us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316281.052864:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322668928:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052870:0:13140:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5369 00000100:00100000:1.0:1761316281.052871:0:9703:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000a254e2e pname:cluuid+ref:pid:xid:nid:opc:job mdt00_003:lustre-MDT0001-lwp-OST0001_UUID+5:3353:x1846869322669056:12345-0@lo:400:kworker/u8:0.0 Request processed in 48us (151us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316281.052876:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000000885db3f pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322669120:12345-0@lo:400:kworker/u8:0.0 Request processed in 27us (111us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316281.052880:0:9703:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5370 00000100:00100000:3.0:1761316281.052881:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1704 00000100:00080000:3.0:1761316281.052915:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316281.052919:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316281.052923:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322668864:0@lo:8:kworker/u8:0.0 00000100:00100000:1.0:1761316281.052939:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f9d9bc94 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-lwp-OST0001_UUID:3353:1846869322669056:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316281.052948:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000671ff9a0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322668992:0@lo:400:kworker/u8:0.0 00010000:02020000:2.0:1761316281.052951:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:0.0:1761316281.052963:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000093968913 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322669120:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316281.052980:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007c56478c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-lwp-MDT0001_UUID:3354:1846869322668736:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316281.052994:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000afc8cc98 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322668928:12345-0@lo:8:kworker/u8:0.0 Request processed in 130us (372us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316281.053002:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2646 00000100:00080000:3.0:1761316281.053009:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761316281.053014:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:3.0:1761316281.053018:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322668928:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761316282.075594:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1761316283.085979:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316283.086080:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316283.086128:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:2.0:1761316283.086168:0:3348:0:(events.c:361:request_in_callback()) peer: 12345-192.168.203.51@tcp (source: 12345-192.168.203.51@tcp) 00000100:00100000:3.0:1761316283.086227:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307591104 00000100:00100000:3.0:1761316283.086281:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 2647 00000100:00100000:3.0:1761316283.086288:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307591104:12345-192.168.203.51@tcp:8:kworker/u8:3.0 00000100:00100000:0.0:1761316283.086387:0:6249:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590912 00000100:00100000:0.0:1761316283.086419:0:6249:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 1959 00000100:00100000:0.0:1761316283.086424:0:6249:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307590912:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:0.0:1761316283.086496:0:6249:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000005d3c4e89 pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0002:803c8c07-d6b3-4fdb-a7b3-4227400366d7+9:2264:x1846869307590912:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 71us (416us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316283.086503:0:6249:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 1959 00000100:00100000:1.0:1761316283.086692:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307591040 00010000:02020000:3.0:1761316283.086710:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 192.168.203.51@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:1.0:1761316283.086734:0:6255:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869307590976 00000100:00100000:3.0:1761316283.086744:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ed1079bc pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:2262:x1846869307591104:12345-192.168.203.51@tcp:8:kworker/u8:3.0 Request processed in 454us (772us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316283.086752:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 2647 00000100:00100000:1.0:1761316283.086771:0:6255:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5371 00000100:00100000:1.0:1761316283.086776:0:6255:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307591040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:3.0:1761316283.086790:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.51@tcp, seq: 5372 00000100:00100000:3.0:1761316283.086797:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307590976:12345-192.168.203.51@tcp:400:kworker/u8:3.0 00000100:00100000:1.0:1761316283.086811:0:6255:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000603a95fb pname:cluuid+ref:pid:xid:nid:opc:job mdt00_001:552c6ffa-be0b-4ca9-86e3-4892e1432c16+5:2264:x1846869307591040:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 33us (682us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316283.086817:0:6255:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5371 00000100:00100000:3.0:1761316283.086832:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000ce0031ff pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:552c6ffa-be0b-4ca9-86e3-4892e1432c16+9:2264:x1846869307590976:12345-192.168.203.51@tcp:400:kworker/u8:3.0 Request processed in 34us (663us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316283.086839:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.51@tcp, seq: 5372 00010000:00010000:2.0:1761316283.100716:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1761316283.613707:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322669184:0@lo:41:osp-pre-1-0.0 00000100:00100000:0.0:1761316283.613771:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316283.613942:0:7266:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669184 00000100:00100000:1.0:1761316283.613998:0:7266:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3342 00000100:00100000:1.0:1761316283.614004:0:7266:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322669184:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:1.0:1761316283.614032:0:7266:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:1.0:1761316283.614299:0:7266:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000004870ba5e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_002:lustre-MDT0000-mdtlov_UUID+5:3355:x1846869322669184:12345-0@lo:41:osp-pre-1-0.0 Request processed in 294us (527us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316283.614306:0:7266:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3342 00000100:00100000:0.0:1761316283.614378:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009fcb6e1b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-mdtlov_UUID:3355:1846869322669184:0@lo:41:osp-pre-1-0.0 00010000:00010000:0.0:1761316284.123490:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316285.147506:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316286.171498:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322669312:0@lo:41:osp-pre-0-1.0 00000100:00100000:2.0:1761316286.171498:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000052a02902 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322669248:0@lo:13:osp-pre-1-1.0 00000100:00080000:0.0:1761316286.171509:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0001-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1761316286.171515:0:3355:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322669440:0@lo:400:kworker/u8:0.0 00000100:00080000:0.0:1761316286.171517:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316286.171521:0:17187:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316286.171528:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0001: connect to NID 0@lo last attempt 4528 00000100:00080000:0.0:1761316286.171531:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0001: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316286.171536:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0001: import 0000000007932437 using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316286.171556:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000028fcbdf6 x1846869322669696/t0(0) o8->lustre-OST0000-osc-MDT0001@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761316286.171570:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316286.171574:0:17187:0:(pinger.c:251:ptlrpc_pinger_process_import()) lustre-MDT0000-mdtlov_UUID->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761316286.171574:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316286.171575:0:3355:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00080000:0.0:1761316286.171580:0:17187:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1761316286.171588:0:17187:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1761316286.171592:0:17187:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-MDT0000: connect to NID 0@lo last attempt 4528 00000100:00080000:0.0:1761316286.171594:0:17187:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-MDT0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1761316286.171598:0:17187:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-MDT0000: import 00000000698b45bb using connection 192.168.203.151@tcp/0@lo 00000100:00100000:0.0:1761316286.171609:0:17187:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000057d0d0b5 x1846869322669760/t0(0) o8->lustre-OST0000-osc-MDT0000@0@lo:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761316286.171616:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322669504:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.171621:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322669568:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316286.171635:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669440 00000100:00100000:2.0:1761316286.171651:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316286.171665:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316286.171680:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000793073e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322669888:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316286.171696:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322669696:0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316286.171700:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669568 00000100:00100000:1.0:1761316286.171706:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005b1075d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322669952:0@lo:400:kworker/u8:0.0 00000100:00100000:2.0:1761316286.171715:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316286.171731:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669888 00000100:00100000:2.0:1761316286.171745:0:3356:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 1 async RPCs [2->3] 00000100:00100000:0.0:1761316286.171749:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316286.171754:0:3356:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322669824:0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316286.171760:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5373 00000100:00100000:1.0:1761316286.171770:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5374 00000100:00100000:3.0:1761316286.171770:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322669440:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.171779:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322669568:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316286.171785:0:3352:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322669760:0@lo:8:kworker/u8:0.0 00000100:00100000:2.0:1761316286.171794:0:3356:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316286.171827:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669312 00000100:00100000:0.0:1761316286.171829:0:3352:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:3.0:1761316286.171843:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000671ff9a0 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-MDT0000_UUID+5:3355:x1846869322669440:12345-0@lo:400:kworker/u8:0.0 Request processed in 74us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.171849:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000af1aef25 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0000-lwp-MDT0001_UUID+5:3353:x1846869322669568:12345-0@lo:400:kworker/u8:0.0 Request processed in 68us (183us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316286.171855:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5373 00000100:00100000:2.0:1761316286.171858:0:17074:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669504 00000100:00100000:1.0:1761316286.171863:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5374 00000100:00100000:3.0:1761316286.171880:0:13921:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669824 00000100:00100000:0.0:1761316286.171885:0:8830:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669248 00000100:00100000:2.0:1761316286.171886:0:17074:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3343 00000100:00100000:1.0:1761316286.171894:0:6254:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5375 00000100:00100000:2.0:1761316286.171894:0:17074:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322669312:12345-0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316286.171902:0:6254:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322669888:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316286.171914:0:13921:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 5376 00000100:00100000:3.0:1761316286.171919:0:13921:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000dfd01862 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322669824:12345-0@lo:400:kworker/u8:0.0 00000004:00000024:2.0:1761316286.171920:0:17074:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446002688 avail 1316540416 00000100:00100000:0.0:1761316286.171927:0:8830:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1130 00000100:00100000:0.0:1761316286.171933:0:8830:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322669248:12345-0@lo:13:osp-pre-1-1.0 00000100:00100000:1.0:1761316286.171943:0:6254:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000b54d8406 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_000:lustre-MDT0001-lwp-OST0001_UUID+5:3356:x1846869322669888:12345-0@lo:400:kworker/u8:0.0 Request processed in 40us (230us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.171951:0:6254:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5375 00002000:00000024:0.0:1761316286.171959:0:8830:0:(ofd_obd.c:740:ofd_statfs()) blocks cached 0 granted 279104 pending 0 free 3923828736 avail 3692306432 00000100:00100000:2.0:1761316286.171976:0:17074:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bf1342a6 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_003:lustre-MDT0001-mdtlov_UUID+7:3353:x1846869322669312:12345-0@lo:41:osp-pre-0-1.0 Request processed in 81us (410us total) trans 0 rc 0/0 00000100:00100000:3.0:1761316286.171979:0:13921:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000dfd01862 pname:cluuid+ref:pid:xid:nid:opc:job mdt00_005:lustre-MDT0000-lwp-OST0001_UUID+5:3356:x1846869322669824:12345-0@lo:400:kworker/u8:0.0 Request processed in 57us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.171984:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3344 00000100:00100000:2.0:1761316286.171993:0:17074:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3343 00000100:00100000:0.0:1761316286.171998:0:8830:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000c64d6e0e pname:cluuid+ref:pid:xid:nid:opc:job ll_ost_create00:lustre-MDT0001-mdtlov_UUID+5:3356:x1846869322669248:12345-0@lo:13:osp-pre-1-1.0 Request processed in 62us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.171998:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001c97fc10 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+7:3356:x1846869322669504:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:3.0:1761316286.171998:0:13921:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 5376 00000100:00100000:0.0:1761316286.172011:0:8830:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1130 00000100:00100000:2.0:1761316286.172025:0:8831:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669696 00000100:00100000:3.0:1761316286.172030:0:8229:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669760 00000100:00100000:1.0:1761316286.172043:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001c97fc10 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0001-mdtlov_UUID+6:3356:x1846869322669504:12345-0@lo:400:kworker/u8:0.0 Request processed in 50us (393us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.172053:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3344 00000100:00100000:2.0:1761316286.172054:0:8831:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2648 00000100:00100000:2.0:1761316286.172062:0:8831:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322669696:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:3.0:1761316286.172063:0:8229:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 2649 00000100:00100000:3.0:1761316286.172080:0:8229:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322669760:12345-0@lo:8:kworker/u8:0.0 00000100:00100000:0.0:1761316286.172097:0:3355:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e9965e9b pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:lustre-MDT0000-lwp-MDT0000_UUID:3355:1846869322669440:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172106:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:1.0:1761316286.172142:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009404a1e5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-lwp-MDT0001_UUID:3353:1846869322669568:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172161:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000ad86ef88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0001-mdtlov_UUID:3353:1846869322669312:0@lo:41:osp-pre-0-1.0 00000100:00100000:1.0:1761316286.172182:0:3353:0:(ptlrpcd.c:415:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1761316286.172187:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322669376:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316286.172201:0:8228:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669952 00000100:00100000:1.0:1761316286.172217:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316286.172230:0:8228:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1705 00000100:00100000:0.0:1761316286.172236:0:8228:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322669952:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172237:0:3353:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000cdfd94f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322669632:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172267:0:3353:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:0.0:1761316286.172281:0:8228:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000e334a674 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_001:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322669952:12345-0@lo:400:kworker/u8:0.0 Request processed in 44us (175us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316286.172288:0:8228:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1705 00000100:00100000:1.0:1761316286.172295:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005b1075d8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322669952:0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316286.172311:0:6259:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669632 00000100:00100000:1.0:1761316286.172327:0:6247:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322669376 00000100:00100000:0.0:1761316286.172337:0:6259:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3345 00000100:00100000:0.0:1761316286.172366:0:6259:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@000000001014ca3e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322669632:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172385:0:6247:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 1960 00000100:00100000:1.0:1761316286.172391:0:6247:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322669376:12345-0@lo:400:kworker/u8:0.0 00000100:00100000:0.0:1761316286.172403:0:6259:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000001014ca3e pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_000:lustre-MDT0000-mdtlov_UUID+5:3353:x1846869322669632:12345-0@lo:400:kworker/u8:0.0 Request processed in 60us (136us total) trans 0 rc 0/0 00000100:00100000:0.0:1761316286.172411:0:6259:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3345 00000100:00100000:1.0:1761316286.172429:0:6247:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000132c270c pname:cluuid+ref:pid:xid:nid:opc:job ll_mgs_0000:557e7bf9-3fe3-4784-8f78-b1773a90f048+20:3353:x1846869322669376:12345-0@lo:400:kworker/u8:0.0 Request processed in 37us (213us total) trans 0 rc 0/0 00000100:00100000:1.0:1761316286.172435:0:6247:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 1960 00000100:00100000:1.0:1761316286.172470:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006e65d9e2 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:557e7bf9-3fe3-4784-8f78-b1773a90f048:3353:1846869322669376:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172478:0:3353:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000cdfd94f8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:lustre-MDT0000-mdtlov_UUID:3353:1846869322669632:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172542:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000375aab8f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0000-lwp-OST0001_UUID:3356:1846869322669824:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172551:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000793073e4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-lwp-OST0001_UUID:3356:1846869322669888:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172558:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005c899dd9 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322669504:0@lo:400:kworker/u8:0.0 00000100:00100000:1.0:1761316286.172569:0:3356:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000052a02902 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_03:lustre-MDT0001-mdtlov_UUID:3356:1846869322669248:0@lo:13:osp-pre-1-1.0 00010000:00010000:0.0:1761316286.172605:0:6241:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:02020000:2.0:1761316286.172796:0:8831:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00010000:02020000:3.0:1761316286.172815:0:8229:0:(ldlm_lib.c:1123:target_handle_connect()) 137-5: lustre-OST0000_UUID: not available for connect from 0@lo (no target). If you are running an HA pair check that the target is mounted on the other server. 00000100:00100000:2.0:1761316286.172844:0:8831:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000007102e769 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_003:0+-99:3352:x1846869322669696:12345-0@lo:8:kworker/u8:0.0 Request processed in 782us (1095us total) trans 0 rc -19/-19 00000100:00100000:3.0:1761316286.172852:0:8229:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@000000002b93b542 pname:cluuid+ref:pid:xid:nid:opc:job ll_ost00_002:0+-99:3352:x1846869322669760:12345-0@lo:8:kworker/u8:0.0 Request processed in 771us (1024us total) trans 0 rc -19/-19 00000100:00100000:2.0:1761316286.172853:0:8831:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2648 00000100:00100000:3.0:1761316286.172863:0:8229:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 2649 00000100:00080000:0.0:1761316286.173052:0:3352:0:(import.c:86:import_set_state_nolock()) 0000000007932437 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316286.173056:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316286.173060:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000028fcbdf6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0001-mdtlov_UUID:3352:1846869322669696:0@lo:8:kworker/u8:0.0 00000100:00080000:0.0:1761316286.173084:0:3352:0:(import.c:86:import_set_state_nolock()) 00000000698b45bb lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1761316286.173087:0:3352:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.151@tcp failed (-19) 00000100:00100000:0.0:1761316286.173089:0:3352:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000057d0d0b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:lustre-MDT0000-mdtlov_UUID:3352:1846869322669760:0@lo:8:kworker/u8:0.0 00010000:00010000:0.0:1761316287.195524:0:7250:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1761316288.219522:0:6242:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1761316288.731721:0:3354:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322670016:0@lo:41:osp-pre-1-0.0 00000100:00100000:1.0:1761316288.731792:0:3354:0:(events.c:361:request_in_callback()) peer: 12345-0@lo (source: 12345-0@lo) 00000100:00100000:2.0:1761316288.731852:0:6260:0:(service.c:2152:ptlrpc_server_handle_req_in()) got req x1846869322670016 00000100:00100000:2.0:1761316288.731909:0:6260:0:(nrs_fifo.c:179:nrs_fifo_req_get()) NRS start fifo request from 12345-0@lo, seq: 3346 00000100:00100000:2.0:1761316288.731915:0:6260:0:(service.c:2324:ptlrpc_server_handle_request()) Handling RPC req@00000000bba19120 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322670016:12345-0@lo:41:osp-pre-1-0.0 00000004:00000024:2.0:1761316288.731947:0:6260:0:(mdt_handler.c:548:mdt_statfs()) blocks cached 0 granted 2146304 pending 0 free 1446215680 avail 1316753408 00000100:00100000:2.0:1761316288.732003:0:6260:0:(service.c:2372:ptlrpc_server_handle_request()) Handled RPC req@00000000bba19120 pname:cluuid+ref:pid:xid:nid:opc:job mdt_out00_001:lustre-MDT0000-mdtlov_UUID+5:3354:x1846869322670016:12345-0@lo:41:osp-pre-1-0.0 Request processed in 87us (211us total) trans 0 rc 0/0 00000100:00100000:2.0:1761316288.732010:0:6260:0:(nrs_fifo.c:241:nrs_fifo_req_stop()) NRS stop fifo request from 12345-0@lo, seq: 3346 00000100:00100000:0.0:1761316288.732149:0:3354:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db207c22 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:lustre-MDT0000-mdtlov_UUID:3354:1846869322670016:0@lo:41:osp-pre-1-0.0 Debug log: 35607 lines, 35607 kept, 0 dropped, 0 bad.